1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5
6 #include <cstring>
7 #include <memory>
8 #include <string>
9 #include "db/version_edit.h"
10 #include "db/version_set.h"
11 #include "logging/logging.h"
12 #include "test_util/testharness.h"
13 #include "test_util/testutil.h"
14 #include "util/string_util.h"
15
16 namespace ROCKSDB_NAMESPACE {
17
18 class VersionBuilderTest : public testing::Test {
19 public:
20 const Comparator* ucmp_;
21 InternalKeyComparator icmp_;
22 Options options_;
23 ImmutableCFOptions ioptions_;
24 MutableCFOptions mutable_cf_options_;
25 VersionStorageInfo vstorage_;
26 uint32_t file_num_;
27 CompactionOptionsFIFO fifo_options_;
28 std::vector<uint64_t> size_being_compacted_;
29
VersionBuilderTest()30 VersionBuilderTest()
31 : ucmp_(BytewiseComparator()),
32 icmp_(ucmp_),
33 ioptions_(options_),
34 mutable_cf_options_(options_),
35 vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
36 nullptr, false),
37 file_num_(1) {
38 mutable_cf_options_.RefreshDerivedOptions(ioptions_);
39 size_being_compacted_.resize(options_.num_levels);
40 }
41
~VersionBuilderTest()42 ~VersionBuilderTest() override {
43 for (int i = 0; i < vstorage_.num_levels(); i++) {
44 for (auto* f : vstorage_.LevelFiles(i)) {
45 if (--f->refs == 0) {
46 delete f;
47 }
48 }
49 }
50 }
51
GetInternalKey(const char * ukey,SequenceNumber smallest_seq=100)52 InternalKey GetInternalKey(const char* ukey,
53 SequenceNumber smallest_seq = 100) {
54 return InternalKey(ukey, smallest_seq, kTypeValue);
55 }
56
Add(int level,uint32_t file_number,const char * smallest,const char * largest,uint64_t file_size=0,uint32_t path_id=0,SequenceNumber smallest_seq=100,SequenceNumber largest_seq=100,uint64_t num_entries=0,uint64_t num_deletions=0,bool sampled=false,SequenceNumber smallest_seqno=0,SequenceNumber largest_seqno=0,uint64_t oldest_blob_file_number=kInvalidBlobFileNumber)57 void Add(int level, uint32_t file_number, const char* smallest,
58 const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
59 SequenceNumber smallest_seq = 100, SequenceNumber largest_seq = 100,
60 uint64_t num_entries = 0, uint64_t num_deletions = 0,
61 bool sampled = false, SequenceNumber smallest_seqno = 0,
62 SequenceNumber largest_seqno = 0,
63 uint64_t oldest_blob_file_number = kInvalidBlobFileNumber) {
64 assert(level < vstorage_.num_levels());
65 FileMetaData* f = new FileMetaData(
66 file_number, path_id, file_size, GetInternalKey(smallest, smallest_seq),
67 GetInternalKey(largest, largest_seq), smallest_seqno, largest_seqno,
68 /* marked_for_compact */ false, oldest_blob_file_number,
69 kUnknownOldestAncesterTime, kUnknownFileCreationTime,
70 kUnknownFileChecksum, kUnknownFileChecksumFuncName);
71 f->compensated_file_size = file_size;
72 f->num_entries = num_entries;
73 f->num_deletions = num_deletions;
74 vstorage_.AddFile(level, f);
75 if (sampled) {
76 f->init_stats_from_file = true;
77 vstorage_.UpdateAccumulatedStats(f);
78 }
79 }
80
AddBlob(uint64_t blob_file_number,uint64_t total_blob_count,uint64_t total_blob_bytes,std::string checksum_method,std::string checksum_value,uint64_t garbage_blob_count,uint64_t garbage_blob_bytes)81 void AddBlob(uint64_t blob_file_number, uint64_t total_blob_count,
82 uint64_t total_blob_bytes, std::string checksum_method,
83 std::string checksum_value, uint64_t garbage_blob_count,
84 uint64_t garbage_blob_bytes) {
85 auto shared_meta = std::make_shared<SharedBlobFileMetaData>(
86 blob_file_number, total_blob_count, total_blob_bytes,
87 std::move(checksum_method), std::move(checksum_value));
88 auto meta = std::make_shared<BlobFileMetaData>(
89 std::move(shared_meta), garbage_blob_count, garbage_blob_bytes);
90
91 vstorage_.AddBlobFile(std::move(meta));
92 }
93
GetBlobFileMetaData(const VersionStorageInfo::BlobFiles & blob_files,uint64_t blob_file_number)94 static std::shared_ptr<BlobFileMetaData> GetBlobFileMetaData(
95 const VersionStorageInfo::BlobFiles& blob_files,
96 uint64_t blob_file_number) {
97 const auto it = blob_files.find(blob_file_number);
98
99 if (it == blob_files.end()) {
100 return std::shared_ptr<BlobFileMetaData>();
101 }
102
103 const auto& meta = it->second;
104 assert(meta);
105
106 return meta;
107 }
108
UpdateVersionStorageInfo()109 void UpdateVersionStorageInfo() {
110 vstorage_.UpdateFilesByCompactionPri(ioptions_.compaction_pri);
111 vstorage_.UpdateNumNonEmptyLevels();
112 vstorage_.GenerateFileIndexer();
113 vstorage_.GenerateLevelFilesBrief();
114 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
115 vstorage_.GenerateLevel0NonOverlapping();
116 vstorage_.SetFinalized();
117 }
118 };
119
UnrefFilesInVersion(VersionStorageInfo * new_vstorage)120 void UnrefFilesInVersion(VersionStorageInfo* new_vstorage) {
121 for (int i = 0; i < new_vstorage->num_levels(); i++) {
122 for (auto* f : new_vstorage->LevelFiles(i)) {
123 if (--f->refs == 0) {
124 delete f;
125 }
126 }
127 }
128 }
129
TEST_F(VersionBuilderTest,ApplyAndSaveTo)130 TEST_F(VersionBuilderTest, ApplyAndSaveTo) {
131 Add(0, 1U, "150", "200", 100U);
132
133 Add(1, 66U, "150", "200", 100U);
134 Add(1, 88U, "201", "300", 100U);
135
136 Add(2, 6U, "150", "179", 100U);
137 Add(2, 7U, "180", "220", 100U);
138 Add(2, 8U, "221", "300", 100U);
139
140 Add(3, 26U, "150", "170", 100U);
141 Add(3, 27U, "171", "179", 100U);
142 Add(3, 28U, "191", "220", 100U);
143 Add(3, 29U, "221", "300", 100U);
144 UpdateVersionStorageInfo();
145
146 VersionEdit version_edit;
147 version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
148 GetInternalKey("350"), 200, 200, false,
149 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
150 kUnknownFileCreationTime, kUnknownFileChecksum,
151 kUnknownFileChecksumFuncName);
152 version_edit.DeleteFile(3, 27U);
153
154 EnvOptions env_options;
155
156 VersionBuilder version_builder(env_options, nullptr, &vstorage_);
157
158 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
159 kCompactionStyleLevel, nullptr, false);
160 version_builder.Apply(&version_edit);
161 version_builder.SaveTo(&new_vstorage);
162
163 ASSERT_EQ(400U, new_vstorage.NumLevelBytes(2));
164 ASSERT_EQ(300U, new_vstorage.NumLevelBytes(3));
165
166 UnrefFilesInVersion(&new_vstorage);
167 }
168
TEST_F(VersionBuilderTest,ApplyAndSaveToDynamic)169 TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) {
170 ioptions_.level_compaction_dynamic_level_bytes = true;
171
172 Add(0, 1U, "150", "200", 100U, 0, 200U, 200U, 0, 0, false, 200U, 200U);
173 Add(0, 88U, "201", "300", 100U, 0, 100U, 100U, 0, 0, false, 100U, 100U);
174
175 Add(4, 6U, "150", "179", 100U);
176 Add(4, 7U, "180", "220", 100U);
177 Add(4, 8U, "221", "300", 100U);
178
179 Add(5, 26U, "150", "170", 100U);
180 Add(5, 27U, "171", "179", 100U);
181 UpdateVersionStorageInfo();
182
183 VersionEdit version_edit;
184 version_edit.AddFile(3, 666, 0, 100U, GetInternalKey("301"),
185 GetInternalKey("350"), 200, 200, false,
186 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
187 kUnknownFileCreationTime, kUnknownFileChecksum,
188 kUnknownFileChecksumFuncName);
189 version_edit.DeleteFile(0, 1U);
190 version_edit.DeleteFile(0, 88U);
191
192 EnvOptions env_options;
193
194 VersionBuilder version_builder(env_options, nullptr, &vstorage_);
195
196 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
197 kCompactionStyleLevel, nullptr, false);
198 version_builder.Apply(&version_edit);
199 version_builder.SaveTo(&new_vstorage);
200
201 ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0));
202 ASSERT_EQ(100U, new_vstorage.NumLevelBytes(3));
203 ASSERT_EQ(300U, new_vstorage.NumLevelBytes(4));
204 ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5));
205
206 UnrefFilesInVersion(&new_vstorage);
207 }
208
TEST_F(VersionBuilderTest,ApplyAndSaveToDynamic2)209 TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) {
210 ioptions_.level_compaction_dynamic_level_bytes = true;
211
212 Add(0, 1U, "150", "200", 100U, 0, 200U, 200U, 0, 0, false, 200U, 200U);
213 Add(0, 88U, "201", "300", 100U, 0, 100U, 100U, 0, 0, false, 100U, 100U);
214
215 Add(4, 6U, "150", "179", 100U);
216 Add(4, 7U, "180", "220", 100U);
217 Add(4, 8U, "221", "300", 100U);
218
219 Add(5, 26U, "150", "170", 100U);
220 Add(5, 27U, "171", "179", 100U);
221 UpdateVersionStorageInfo();
222
223 VersionEdit version_edit;
224 version_edit.AddFile(4, 666, 0, 100U, GetInternalKey("301"),
225 GetInternalKey("350"), 200, 200, false,
226 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
227 kUnknownFileCreationTime, kUnknownFileChecksum,
228 kUnknownFileChecksumFuncName);
229 version_edit.DeleteFile(0, 1U);
230 version_edit.DeleteFile(0, 88U);
231 version_edit.DeleteFile(4, 6U);
232 version_edit.DeleteFile(4, 7U);
233 version_edit.DeleteFile(4, 8U);
234
235 EnvOptions env_options;
236
237 VersionBuilder version_builder(env_options, nullptr, &vstorage_);
238
239 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
240 kCompactionStyleLevel, nullptr, false);
241 version_builder.Apply(&version_edit);
242 version_builder.SaveTo(&new_vstorage);
243
244 ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0));
245 ASSERT_EQ(100U, new_vstorage.NumLevelBytes(4));
246 ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5));
247
248 UnrefFilesInVersion(&new_vstorage);
249 }
250
TEST_F(VersionBuilderTest,ApplyMultipleAndSaveTo)251 TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) {
252 UpdateVersionStorageInfo();
253
254 VersionEdit version_edit;
255 version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
256 GetInternalKey("350"), 200, 200, false,
257 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
258 kUnknownFileCreationTime, kUnknownFileChecksum,
259 kUnknownFileChecksumFuncName);
260 version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
261 GetInternalKey("450"), 200, 200, false,
262 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
263 kUnknownFileCreationTime, kUnknownFileChecksum,
264 kUnknownFileChecksumFuncName);
265 version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
266 GetInternalKey("650"), 200, 200, false,
267 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
268 kUnknownFileCreationTime, kUnknownFileChecksum,
269 kUnknownFileChecksumFuncName);
270 version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
271 GetInternalKey("550"), 200, 200, false,
272 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
273 kUnknownFileCreationTime, kUnknownFileChecksum,
274 kUnknownFileChecksumFuncName);
275 version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
276 GetInternalKey("750"), 200, 200, false,
277 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
278 kUnknownFileCreationTime, kUnknownFileChecksum,
279 kUnknownFileChecksumFuncName);
280
281 EnvOptions env_options;
282
283 VersionBuilder version_builder(env_options, nullptr, &vstorage_);
284
285 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
286 kCompactionStyleLevel, nullptr, false);
287 version_builder.Apply(&version_edit);
288 version_builder.SaveTo(&new_vstorage);
289
290 ASSERT_EQ(500U, new_vstorage.NumLevelBytes(2));
291
292 UnrefFilesInVersion(&new_vstorage);
293 }
294
TEST_F(VersionBuilderTest,ApplyDeleteAndSaveTo)295 TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
296 UpdateVersionStorageInfo();
297
298 EnvOptions env_options;
299 VersionBuilder version_builder(env_options, nullptr, &vstorage_);
300 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
301 kCompactionStyleLevel, nullptr, false);
302
303 VersionEdit version_edit;
304 version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
305 GetInternalKey("350"), 200, 200, false,
306 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
307 kUnknownFileCreationTime, kUnknownFileChecksum,
308 kUnknownFileChecksumFuncName);
309 version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
310 GetInternalKey("450"), 200, 200, false,
311 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
312 kUnknownFileCreationTime, kUnknownFileChecksum,
313 kUnknownFileChecksumFuncName);
314 version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
315 GetInternalKey("650"), 200, 200, false,
316 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
317 kUnknownFileCreationTime, kUnknownFileChecksum,
318 kUnknownFileChecksumFuncName);
319 version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
320 GetInternalKey("550"), 200, 200, false,
321 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
322 kUnknownFileCreationTime, kUnknownFileChecksum,
323 kUnknownFileChecksumFuncName);
324 version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
325 GetInternalKey("750"), 200, 200, false,
326 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
327 kUnknownFileCreationTime, kUnknownFileChecksum,
328 kUnknownFileChecksumFuncName);
329 version_builder.Apply(&version_edit);
330
331 VersionEdit version_edit2;
332 version_edit.AddFile(2, 808, 0, 100U, GetInternalKey("901"),
333 GetInternalKey("950"), 200, 200, false,
334 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
335 kUnknownFileCreationTime, kUnknownFileChecksum,
336 kUnknownFileChecksumFuncName);
337 version_edit2.DeleteFile(2, 616);
338 version_edit2.DeleteFile(2, 636);
339 version_edit.AddFile(2, 806, 0, 100U, GetInternalKey("801"),
340 GetInternalKey("850"), 200, 200, false,
341 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
342 kUnknownFileCreationTime, kUnknownFileChecksum,
343 kUnknownFileChecksumFuncName);
344 version_builder.Apply(&version_edit2);
345
346 version_builder.SaveTo(&new_vstorage);
347
348 ASSERT_EQ(300U, new_vstorage.NumLevelBytes(2));
349
350 UnrefFilesInVersion(&new_vstorage);
351 }
352
TEST_F(VersionBuilderTest,ApplyBlobFileAddition)353 TEST_F(VersionBuilderTest, ApplyBlobFileAddition) {
354 EnvOptions env_options;
355 constexpr TableCache* table_cache = nullptr;
356 VersionBuilder builder(env_options, table_cache, &vstorage_);
357
358 VersionEdit edit;
359
360 constexpr uint64_t blob_file_number = 1234;
361 constexpr uint64_t total_blob_count = 5678;
362 constexpr uint64_t total_blob_bytes = 999999;
363 constexpr char checksum_method[] = "SHA1";
364 constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd";
365
366 edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
367 checksum_method, checksum_value);
368
369 ASSERT_OK(builder.Apply(&edit));
370
371 constexpr bool force_consistency_checks = false;
372 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
373 kCompactionStyleLevel, &vstorage_,
374 force_consistency_checks);
375
376 ASSERT_OK(builder.SaveTo(&new_vstorage));
377
378 const auto& new_blob_files = new_vstorage.GetBlobFiles();
379 ASSERT_EQ(new_blob_files.size(), 1);
380
381 const auto new_meta = GetBlobFileMetaData(new_blob_files, blob_file_number);
382
383 ASSERT_NE(new_meta, nullptr);
384 ASSERT_EQ(new_meta->GetBlobFileNumber(), blob_file_number);
385 ASSERT_EQ(new_meta->GetTotalBlobCount(), total_blob_count);
386 ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes);
387 ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method);
388 ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value);
389 ASSERT_EQ(new_meta->GetGarbageBlobCount(), 0);
390 ASSERT_EQ(new_meta->GetGarbageBlobBytes(), 0);
391 }
392
TEST_F(VersionBuilderTest,ApplyBlobFileAdditionAlreadyInBase)393 TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyInBase) {
394 // Attempt to add a blob file that is already present in the base version.
395
396 constexpr uint64_t blob_file_number = 1234;
397 constexpr uint64_t total_blob_count = 5678;
398 constexpr uint64_t total_blob_bytes = 999999;
399 constexpr char checksum_method[] = "SHA1";
400 constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd";
401 constexpr uint64_t garbage_blob_count = 123;
402 constexpr uint64_t garbage_blob_bytes = 456789;
403
404 AddBlob(blob_file_number, total_blob_count, total_blob_bytes, checksum_method,
405 checksum_value, garbage_blob_count, garbage_blob_bytes);
406
407 EnvOptions env_options;
408 constexpr TableCache* table_cache = nullptr;
409 VersionBuilder builder(env_options, table_cache, &vstorage_);
410
411 VersionEdit edit;
412
413 edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
414 checksum_method, checksum_value);
415
416 const Status s = builder.Apply(&edit);
417 ASSERT_TRUE(s.IsCorruption());
418 ASSERT_TRUE(std::strstr(s.getState(), "Blob file #1234 already added"));
419 }
420
TEST_F(VersionBuilderTest,ApplyBlobFileAdditionAlreadyApplied)421 TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyApplied) {
422 // Attempt to add the same blob file twice using version edits.
423
424 EnvOptions env_options;
425 constexpr TableCache* table_cache = nullptr;
426 VersionBuilder builder(env_options, table_cache, &vstorage_);
427
428 VersionEdit edit;
429
430 constexpr uint64_t blob_file_number = 1234;
431 constexpr uint64_t total_blob_count = 5678;
432 constexpr uint64_t total_blob_bytes = 999999;
433 constexpr char checksum_method[] = "SHA1";
434 constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd";
435
436 edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
437 checksum_method, checksum_value);
438
439 ASSERT_OK(builder.Apply(&edit));
440
441 const Status s = builder.Apply(&edit);
442 ASSERT_TRUE(s.IsCorruption());
443 ASSERT_TRUE(std::strstr(s.getState(), "Blob file #1234 already added"));
444 }
445
TEST_F(VersionBuilderTest,ApplyBlobFileGarbageFileInBase)446 TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) {
447 // Increase the amount of garbage for a blob file present in the base version.
448
449 constexpr uint64_t blob_file_number = 1234;
450 constexpr uint64_t total_blob_count = 5678;
451 constexpr uint64_t total_blob_bytes = 999999;
452 constexpr char checksum_method[] = "SHA1";
453 constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd";
454 constexpr uint64_t garbage_blob_count = 123;
455 constexpr uint64_t garbage_blob_bytes = 456789;
456
457 AddBlob(blob_file_number, total_blob_count, total_blob_bytes, checksum_method,
458 checksum_value, garbage_blob_count, garbage_blob_bytes);
459
460 const auto meta =
461 GetBlobFileMetaData(vstorage_.GetBlobFiles(), blob_file_number);
462 ASSERT_NE(meta, nullptr);
463
464 EnvOptions env_options;
465 constexpr TableCache* table_cache = nullptr;
466 VersionBuilder builder(env_options, table_cache, &vstorage_);
467
468 VersionEdit edit;
469
470 constexpr uint64_t new_garbage_blob_count = 456;
471 constexpr uint64_t new_garbage_blob_bytes = 111111;
472
473 edit.AddBlobFileGarbage(blob_file_number, new_garbage_blob_count,
474 new_garbage_blob_bytes);
475
476 ASSERT_OK(builder.Apply(&edit));
477
478 constexpr bool force_consistency_checks = false;
479 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
480 kCompactionStyleLevel, &vstorage_,
481 force_consistency_checks);
482
483 ASSERT_OK(builder.SaveTo(&new_vstorage));
484
485 const auto& new_blob_files = new_vstorage.GetBlobFiles();
486 ASSERT_EQ(new_blob_files.size(), 1);
487
488 const auto new_meta = GetBlobFileMetaData(new_blob_files, blob_file_number);
489
490 ASSERT_NE(new_meta, nullptr);
491 ASSERT_EQ(new_meta->GetSharedMeta(), meta->GetSharedMeta());
492 ASSERT_EQ(new_meta->GetBlobFileNumber(), blob_file_number);
493 ASSERT_EQ(new_meta->GetTotalBlobCount(), total_blob_count);
494 ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes);
495 ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method);
496 ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value);
497 ASSERT_EQ(new_meta->GetGarbageBlobCount(),
498 garbage_blob_count + new_garbage_blob_count);
499 ASSERT_EQ(new_meta->GetGarbageBlobBytes(),
500 garbage_blob_bytes + new_garbage_blob_bytes);
501 }
502
TEST_F(VersionBuilderTest,ApplyBlobFileGarbageFileAdditionApplied)503 TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileAdditionApplied) {
504 // Increase the amount of garbage for a blob file added using a version edit.
505
506 EnvOptions env_options;
507 constexpr TableCache* table_cache = nullptr;
508 VersionBuilder builder(env_options, table_cache, &vstorage_);
509
510 VersionEdit addition;
511
512 constexpr uint64_t blob_file_number = 1234;
513 constexpr uint64_t total_blob_count = 5678;
514 constexpr uint64_t total_blob_bytes = 999999;
515 constexpr char checksum_method[] = "SHA1";
516 constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd";
517
518 addition.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
519 checksum_method, checksum_value);
520
521 ASSERT_OK(builder.Apply(&addition));
522
523 constexpr uint64_t garbage_blob_count = 123;
524 constexpr uint64_t garbage_blob_bytes = 456789;
525
526 VersionEdit garbage;
527
528 garbage.AddBlobFileGarbage(blob_file_number, garbage_blob_count,
529 garbage_blob_bytes);
530
531 ASSERT_OK(builder.Apply(&garbage));
532
533 constexpr bool force_consistency_checks = false;
534 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
535 kCompactionStyleLevel, &vstorage_,
536 force_consistency_checks);
537
538 ASSERT_OK(builder.SaveTo(&new_vstorage));
539
540 const auto& new_blob_files = new_vstorage.GetBlobFiles();
541 ASSERT_EQ(new_blob_files.size(), 1);
542
543 const auto new_meta = GetBlobFileMetaData(new_blob_files, blob_file_number);
544
545 ASSERT_NE(new_meta, nullptr);
546 ASSERT_EQ(new_meta->GetBlobFileNumber(), blob_file_number);
547 ASSERT_EQ(new_meta->GetTotalBlobCount(), total_blob_count);
548 ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes);
549 ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method);
550 ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value);
551 ASSERT_EQ(new_meta->GetGarbageBlobCount(), garbage_blob_count);
552 ASSERT_EQ(new_meta->GetGarbageBlobBytes(), garbage_blob_bytes);
553 }
554
TEST_F(VersionBuilderTest,ApplyBlobFileGarbageFileNotFound)555 TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileNotFound) {
556 // Attempt to increase the amount of garbage for a blob file that is
557 // neither in the base version, nor was it added using a version edit.
558
559 EnvOptions env_options;
560 constexpr TableCache* table_cache = nullptr;
561 VersionBuilder builder(env_options, table_cache, &vstorage_);
562
563 VersionEdit edit;
564
565 constexpr uint64_t blob_file_number = 1234;
566 constexpr uint64_t garbage_blob_count = 5678;
567 constexpr uint64_t garbage_blob_bytes = 999999;
568
569 edit.AddBlobFileGarbage(blob_file_number, garbage_blob_count,
570 garbage_blob_bytes);
571
572 const Status s = builder.Apply(&edit);
573 ASSERT_TRUE(s.IsCorruption());
574 ASSERT_TRUE(std::strstr(s.getState(), "Blob file #1234 not found"));
575 }
576
TEST_F(VersionBuilderTest,SaveBlobFilesTo)577 TEST_F(VersionBuilderTest, SaveBlobFilesTo) {
578 // Add three blob files to base version.
579 for (uint64_t i = 1; i <= 3; ++i) {
580 const uint64_t blob_file_number = i;
581 const uint64_t total_blob_count = i * 1000;
582 const uint64_t total_blob_bytes = i * 1000000;
583 const uint64_t garbage_blob_count = i * 100;
584 const uint64_t garbage_blob_bytes = i * 20000;
585
586 AddBlob(blob_file_number, total_blob_count, total_blob_bytes,
587 /* checksum_method */ std::string(),
588 /* checksum_value */ std::string(), garbage_blob_count,
589 garbage_blob_bytes);
590 }
591
592 EnvOptions env_options;
593 constexpr TableCache* table_cache = nullptr;
594 VersionBuilder builder(env_options, table_cache, &vstorage_);
595
596 VersionEdit edit;
597
598 // Add some garbage to the second and third blob files. The second blob file
599 // remains valid since it does not consist entirely of garbage yet. The third
600 // blob file is all garbage after the edit and will not be part of the new
601 // version.
602 edit.AddBlobFileGarbage(/* blob_file_number */ 2,
603 /* garbage_blob_count */ 200,
604 /* garbage_blob_bytes */ 100000);
605 edit.AddBlobFileGarbage(/* blob_file_number */ 3,
606 /* garbage_blob_count */ 2700,
607 /* garbage_blob_bytes */ 2940000);
608
609 // Add a fourth blob file.
610 edit.AddBlobFile(/* blob_file_number */ 4, /* total_blob_count */ 4000,
611 /* total_blob_bytes */ 4000000,
612 /* checksum_method */ std::string(),
613 /* checksum_value */ std::string());
614
615 ASSERT_OK(builder.Apply(&edit));
616
617 constexpr bool force_consistency_checks = false;
618 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
619 kCompactionStyleLevel, &vstorage_,
620 force_consistency_checks);
621
622 ASSERT_OK(builder.SaveTo(&new_vstorage));
623
624 const auto& new_blob_files = new_vstorage.GetBlobFiles();
625 ASSERT_EQ(new_blob_files.size(), 3);
626
627 const auto meta1 = GetBlobFileMetaData(new_blob_files, 1);
628
629 ASSERT_NE(meta1, nullptr);
630 ASSERT_EQ(meta1->GetBlobFileNumber(), 1);
631 ASSERT_EQ(meta1->GetTotalBlobCount(), 1000);
632 ASSERT_EQ(meta1->GetTotalBlobBytes(), 1000000);
633 ASSERT_EQ(meta1->GetGarbageBlobCount(), 100);
634 ASSERT_EQ(meta1->GetGarbageBlobBytes(), 20000);
635
636 const auto meta2 = GetBlobFileMetaData(new_blob_files, 2);
637
638 ASSERT_NE(meta2, nullptr);
639 ASSERT_EQ(meta2->GetBlobFileNumber(), 2);
640 ASSERT_EQ(meta2->GetTotalBlobCount(), 2000);
641 ASSERT_EQ(meta2->GetTotalBlobBytes(), 2000000);
642 ASSERT_EQ(meta2->GetGarbageBlobCount(), 400);
643 ASSERT_EQ(meta2->GetGarbageBlobBytes(), 140000);
644
645 const auto meta4 = GetBlobFileMetaData(new_blob_files, 4);
646
647 ASSERT_NE(meta4, nullptr);
648 ASSERT_EQ(meta4->GetBlobFileNumber(), 4);
649 ASSERT_EQ(meta4->GetTotalBlobCount(), 4000);
650 ASSERT_EQ(meta4->GetTotalBlobBytes(), 4000000);
651 ASSERT_EQ(meta4->GetGarbageBlobCount(), 0);
652 ASSERT_EQ(meta4->GetGarbageBlobBytes(), 0);
653 }
654
TEST_F(VersionBuilderTest,CheckConsistencyForBlobFiles)655 TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
656 // Initialize base version. The first table file points to a valid blob file
657 // in this version; the second one does not refer to any blob files.
658
659 Add(/* level */ 1, /* file_number */ 1, /* smallest */ "150",
660 /* largest */ "200", /* file_size */ 100,
661 /* path_id */ 0, /* smallest_seq */ 100, /* largest_seq */ 100,
662 /* num_entries */ 0, /* num_deletions */ 0,
663 /* sampled */ false, /* smallest_seqno */ 100, /* largest_seqno */ 100,
664 /* oldest_blob_file_number */ 16);
665 Add(/* level */ 1, /* file_number */ 23, /* smallest */ "201",
666 /* largest */ "300", /* file_size */ 100,
667 /* path_id */ 0, /* smallest_seq */ 200, /* largest_seq */ 200,
668 /* num_entries */ 0, /* num_deletions */ 0,
669 /* sampled */ false, /* smallest_seqno */ 200, /* largest_seqno */ 200,
670 kInvalidBlobFileNumber);
671
672 AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000,
673 /* total_blob_bytes */ 1000000,
674 /* checksum_method */ std::string(),
675 /* checksum_value */ std::string(),
676 /* garbage_blob_count */ 500, /* garbage_blob_bytes */ 300000);
677
678 UpdateVersionStorageInfo();
679
680 // Add a new table file that points to the existing blob file, and add a
681 // new table file--blob file pair.
682 EnvOptions env_options;
683 constexpr TableCache* table_cache = nullptr;
684 VersionBuilder builder(env_options, table_cache, &vstorage_);
685
686 VersionEdit edit;
687
688 edit.AddFile(/* level */ 1, /* file_number */ 606, /* path_id */ 0,
689 /* file_size */ 100, /* smallest */ GetInternalKey("701"),
690 /* largest */ GetInternalKey("750"), /* smallest_seqno */ 200,
691 /* largest_seqno */ 200, /* marked_for_compaction */ false,
692 /* oldest_blob_file_number */ 16, kUnknownOldestAncesterTime,
693 kUnknownFileCreationTime, kUnknownFileChecksum,
694 kUnknownFileChecksumFuncName);
695
696 edit.AddFile(/* level */ 1, /* file_number */ 700, /* path_id */ 0,
697 /* file_size */ 100, /* smallest */ GetInternalKey("801"),
698 /* largest */ GetInternalKey("850"), /* smallest_seqno */ 200,
699 /* largest_seqno */ 200, /* marked_for_compaction */ false,
700 /* oldest_blob_file_number */ 1000, kUnknownOldestAncesterTime,
701 kUnknownFileCreationTime, kUnknownFileChecksum,
702 kUnknownFileChecksumFuncName);
703 edit.AddBlobFile(/* blob_file_number */ 1000, /* total_blob_count */ 2000,
704 /* total_blob_bytes */ 200000,
705 /* checksum_method */ std::string(),
706 /* checksum_value */ std::string());
707
708 ASSERT_OK(builder.Apply(&edit));
709
710 // Save to a new version in order to trigger consistency checks.
711 constexpr bool force_consistency_checks = true;
712 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
713 kCompactionStyleLevel, &vstorage_,
714 force_consistency_checks);
715
716 ASSERT_OK(builder.SaveTo(&new_vstorage));
717
718 UnrefFilesInVersion(&new_vstorage);
719 }
720
TEST_F(VersionBuilderTest,CheckConsistencyForBlobFilesNotInVersion)721 TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesNotInVersion) {
722 // Initialize base version. The table file points to a blob file that is
723 // not in this version.
724
725 Add(/* level */ 1, /* file_number */ 1, /* smallest */ "150",
726 /* largest */ "200", /* file_size */ 100,
727 /* path_id */ 0, /* smallest_seq */ 100, /* largest_seq */ 100,
728 /* num_entries */ 0, /* num_deletions */ 0,
729 /* sampled */ false, /* smallest_seqno */ 100, /* largest_seqno */ 100,
730 /* oldest_blob_file_number */ 256);
731
732 AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000,
733 /* total_blob_bytes */ 1000000,
734 /* checksum_method */ std::string(),
735 /* checksum_value */ std::string(),
736 /* garbage_blob_count */ 500, /* garbage_blob_bytes */ 300000);
737
738 UpdateVersionStorageInfo();
739
740 EnvOptions env_options;
741 constexpr TableCache* table_cache = nullptr;
742 VersionBuilder builder(env_options, table_cache, &vstorage_);
743
744 // Save to a new version in order to trigger consistency checks.
745 constexpr bool force_consistency_checks = true;
746 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
747 kCompactionStyleLevel, &vstorage_,
748 force_consistency_checks);
749
750 const Status s = builder.SaveTo(&new_vstorage);
751 ASSERT_TRUE(s.IsCorruption());
752 ASSERT_TRUE(
753 std::strstr(s.getState(), "Blob file #256 is not part of this version"));
754
755 UnrefFilesInVersion(&new_vstorage);
756 }
757
TEST_F(VersionBuilderTest,CheckConsistencyForBlobFilesAllGarbage)758 TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbage) {
759 // Initialize base version. The table file points to a blob file that is
760 // all garbage.
761
762 Add(/* level */ 1, /* file_number */ 1, /* smallest */ "150",
763 /* largest */ "200", /* file_size */ 100,
764 /* path_id */ 0, /* smallest_seq */ 100, /* largest_seq */ 100,
765 /* num_entries */ 0, /* num_deletions */ 0,
766 /* sampled */ false, /* smallest_seqno */ 100, /* largest_seqno */ 100,
767 /* oldest_blob_file_number */ 16);
768
769 AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000,
770 /* total_blob_bytes */ 1000000,
771 /* checksum_method */ std::string(),
772 /* checksum_value */ std::string(),
773 /* garbage_blob_count */ 1000, /* garbage_blob_bytes */ 1000000);
774
775 UpdateVersionStorageInfo();
776
777 EnvOptions env_options;
778 constexpr TableCache* table_cache = nullptr;
779 VersionBuilder builder(env_options, table_cache, &vstorage_);
780
781 // Save to a new version in order to trigger consistency checks.
782 constexpr bool force_consistency_checks = true;
783 VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
784 kCompactionStyleLevel, &vstorage_,
785 force_consistency_checks);
786
787 const Status s = builder.SaveTo(&new_vstorage);
788 ASSERT_TRUE(s.IsCorruption());
789 ASSERT_TRUE(
790 std::strstr(s.getState(), "Blob file #16 consists entirely of garbage"));
791
792 UnrefFilesInVersion(&new_vstorage);
793 }
794
TEST_F(VersionBuilderTest,EstimatedActiveKeys)795 TEST_F(VersionBuilderTest, EstimatedActiveKeys) {
796 const uint32_t kTotalSamples = 20;
797 const uint32_t kNumLevels = 5;
798 const uint32_t kFilesPerLevel = 8;
799 const uint32_t kNumFiles = kNumLevels * kFilesPerLevel;
800 const uint32_t kEntriesPerFile = 1000;
801 const uint32_t kDeletionsPerFile = 100;
802 for (uint32_t i = 0; i < kNumFiles; ++i) {
803 Add(static_cast<int>(i / kFilesPerLevel), i + 1,
804 ToString((i + 100) * 1000).c_str(),
805 ToString((i + 100) * 1000 + 999).c_str(),
806 100U, 0, 100, 100,
807 kEntriesPerFile, kDeletionsPerFile,
808 (i < kTotalSamples));
809 }
810 // minus 2X for the number of deletion entries because:
811 // 1x for deletion entry does not count as a data entry.
812 // 1x for each deletion entry will actually remove one data entry.
813 ASSERT_EQ(vstorage_.GetEstimatedActiveKeys(),
814 (kEntriesPerFile - 2 * kDeletionsPerFile) * kNumFiles);
815 }
816
817 } // namespace ROCKSDB_NAMESPACE
818
main(int argc,char ** argv)819 int main(int argc, char** argv) {
820 ::testing::InitGoogleTest(&argc, argv);
821 return RUN_ALL_TESTS();
822 }
823