1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #ifndef ROCKSDB_LITE
11
12 #include <stdlib.h>
13 #include <map>
14 #include <string>
15 #include <vector>
16 #include "db/db_impl/db_impl.h"
17 #include "db/db_test_util.h"
18 #include "db/version_set.h"
19 #include "db/write_batch_internal.h"
20 #include "file/filename.h"
21 #include "port/stack_trace.h"
22 #include "rocksdb/db.h"
23 #include "rocksdb/env.h"
24 #include "rocksdb/transaction_log.h"
25 #include "test_util/sync_point.h"
26 #include "test_util/testharness.h"
27 #include "test_util/testutil.h"
28 #include "util/string_util.h"
29
30 namespace ROCKSDB_NAMESPACE {
31
32 class DeleteFileTest : public DBTestBase {
33 public:
34 const int numlevels_;
35 const std::string wal_dir_;
36
DeleteFileTest()37 DeleteFileTest()
38 : DBTestBase("/deletefile_test"),
39 numlevels_(7),
40 wal_dir_(dbname_ + "/wal_files") {}
41
SetOptions(Options * options)42 void SetOptions(Options* options) {
43 assert(options);
44 options->delete_obsolete_files_period_micros = 0; // always do full purge
45 options->enable_thread_tracking = true;
46 options->write_buffer_size = 1024 * 1024 * 1000;
47 options->target_file_size_base = 1024 * 1024 * 1000;
48 options->max_bytes_for_level_base = 1024 * 1024 * 1000;
49 options->WAL_ttl_seconds = 300; // Used to test log files
50 options->WAL_size_limit_MB = 1024; // Used to test log files
51 options->wal_dir = wal_dir_;
52 }
53
AddKeys(int numkeys,int startkey=0)54 void AddKeys(int numkeys, int startkey = 0) {
55 WriteOptions options;
56 options.sync = false;
57 ReadOptions roptions;
58 for (int i = startkey; i < (numkeys + startkey) ; i++) {
59 std::string temp = ToString(i);
60 Slice key(temp);
61 Slice value(temp);
62 ASSERT_OK(db_->Put(options, key, value));
63 }
64 }
65
numKeysInLevels(std::vector<LiveFileMetaData> & metadata,std::vector<int> * keysperlevel=nullptr)66 int numKeysInLevels(
67 std::vector<LiveFileMetaData> &metadata,
68 std::vector<int> *keysperlevel = nullptr) {
69
70 if (keysperlevel != nullptr) {
71 keysperlevel->resize(numlevels_);
72 }
73
74 int numKeys = 0;
75 for (size_t i = 0; i < metadata.size(); i++) {
76 int startkey = atoi(metadata[i].smallestkey.c_str());
77 int endkey = atoi(metadata[i].largestkey.c_str());
78 int numkeysinfile = (endkey - startkey + 1);
79 numKeys += numkeysinfile;
80 if (keysperlevel != nullptr) {
81 (*keysperlevel)[(int)metadata[i].level] += numkeysinfile;
82 }
83 fprintf(stderr, "level %d name %s smallest %s largest %s\n",
84 metadata[i].level, metadata[i].name.c_str(),
85 metadata[i].smallestkey.c_str(),
86 metadata[i].largestkey.c_str());
87 }
88 return numKeys;
89 }
90
CreateTwoLevels()91 void CreateTwoLevels() {
92 AddKeys(50000, 10000);
93 ASSERT_OK(dbfull()->TEST_FlushMemTable());
94 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
95 for (int i = 0; i < 2; ++i) {
96 ASSERT_OK(dbfull()->TEST_CompactRange(i, nullptr, nullptr));
97 }
98
99 AddKeys(50000, 10000);
100 ASSERT_OK(dbfull()->TEST_FlushMemTable());
101 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
102 ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
103 }
104
CheckFileTypeCounts(const std::string & dir,int required_log,int required_sst,int required_manifest)105 void CheckFileTypeCounts(const std::string& dir, int required_log,
106 int required_sst, int required_manifest) {
107 std::vector<std::string> filenames;
108 env_->GetChildren(dir, &filenames);
109
110 int log_cnt = 0, sst_cnt = 0, manifest_cnt = 0;
111 for (auto file : filenames) {
112 uint64_t number;
113 FileType type;
114 if (ParseFileName(file, &number, &type)) {
115 log_cnt += (type == kLogFile);
116 sst_cnt += (type == kTableFile);
117 manifest_cnt += (type == kDescriptorFile);
118 }
119 }
120 ASSERT_EQ(required_log, log_cnt);
121 ASSERT_EQ(required_sst, sst_cnt);
122 ASSERT_EQ(required_manifest, manifest_cnt);
123 }
124
DoSleep(void * arg)125 static void DoSleep(void* arg) {
126 auto test = reinterpret_cast<DeleteFileTest*>(arg);
127 test->env_->SleepForMicroseconds(2 * 1000 * 1000);
128 }
129
130 // An empty job to guard all jobs are processed
GuardFinish(void *)131 static void GuardFinish(void* /*arg*/) {
132 TEST_SYNC_POINT("DeleteFileTest::GuardFinish");
133 }
134 };
135
TEST_F(DeleteFileTest,AddKeysAndQueryLevels)136 TEST_F(DeleteFileTest, AddKeysAndQueryLevels) {
137 Options options = CurrentOptions();
138 SetOptions(&options);
139 Destroy(options);
140 options.create_if_missing = true;
141 Reopen(options);
142
143 CreateTwoLevels();
144 std::vector<LiveFileMetaData> metadata;
145 db_->GetLiveFilesMetaData(&metadata);
146
147 std::string level1file = "";
148 int level1keycount = 0;
149 std::string level2file = "";
150 int level2keycount = 0;
151 int level1index = 0;
152 int level2index = 1;
153
154 ASSERT_EQ((int)metadata.size(), 2);
155 if (metadata[0].level == 2) {
156 level1index = 1;
157 level2index = 0;
158 }
159
160 level1file = metadata[level1index].name;
161 int startkey = atoi(metadata[level1index].smallestkey.c_str());
162 int endkey = atoi(metadata[level1index].largestkey.c_str());
163 level1keycount = (endkey - startkey + 1);
164 level2file = metadata[level2index].name;
165 startkey = atoi(metadata[level2index].smallestkey.c_str());
166 endkey = atoi(metadata[level2index].largestkey.c_str());
167 level2keycount = (endkey - startkey + 1);
168
169 // COntrolled setup. Levels 1 and 2 should both have 50K files.
170 // This is a little fragile as it depends on the current
171 // compaction heuristics.
172 ASSERT_EQ(level1keycount, 50000);
173 ASSERT_EQ(level2keycount, 50000);
174
175 Status status = db_->DeleteFile("0.sst");
176 ASSERT_TRUE(status.IsInvalidArgument());
177
178 // intermediate level files cannot be deleted.
179 status = db_->DeleteFile(level1file);
180 ASSERT_TRUE(status.IsInvalidArgument());
181
182 // Lowest level file deletion should succeed.
183 ASSERT_OK(db_->DeleteFile(level2file));
184 }
185
TEST_F(DeleteFileTest,PurgeObsoleteFilesTest)186 TEST_F(DeleteFileTest, PurgeObsoleteFilesTest) {
187 Options options = CurrentOptions();
188 SetOptions(&options);
189 Destroy(options);
190 options.create_if_missing = true;
191 Reopen(options);
192
193 CreateTwoLevels();
194 // there should be only one (empty) log file because CreateTwoLevels()
195 // flushes the memtables to disk
196 CheckFileTypeCounts(wal_dir_, 1, 0, 0);
197 // 2 ssts, 1 manifest
198 CheckFileTypeCounts(dbname_, 0, 2, 1);
199 std::string first("0"), last("999999");
200 CompactRangeOptions compact_options;
201 compact_options.change_level = true;
202 compact_options.target_level = 2;
203 Slice first_slice(first), last_slice(last);
204 db_->CompactRange(compact_options, &first_slice, &last_slice);
205 // 1 sst after compaction
206 CheckFileTypeCounts(dbname_, 0, 1, 1);
207
208 // this time, we keep an iterator alive
209 Reopen(options);
210 Iterator *itr = nullptr;
211 CreateTwoLevels();
212 itr = db_->NewIterator(ReadOptions());
213 db_->CompactRange(compact_options, &first_slice, &last_slice);
214 // 3 sst after compaction with live iterator
215 CheckFileTypeCounts(dbname_, 0, 3, 1);
216 delete itr;
217 // 1 sst after iterator deletion
218 CheckFileTypeCounts(dbname_, 0, 1, 1);
219 }
220
TEST_F(DeleteFileTest,BackgroundPurgeIteratorTest)221 TEST_F(DeleteFileTest, BackgroundPurgeIteratorTest) {
222 Options options = CurrentOptions();
223 SetOptions(&options);
224 Destroy(options);
225 options.create_if_missing = true;
226 Reopen(options);
227
228 std::string first("0"), last("999999");
229 CompactRangeOptions compact_options;
230 compact_options.change_level = true;
231 compact_options.target_level = 2;
232 Slice first_slice(first), last_slice(last);
233
234 // We keep an iterator alive
235 Iterator* itr = nullptr;
236 CreateTwoLevels();
237 ReadOptions read_options;
238 read_options.background_purge_on_iterator_cleanup = true;
239 itr = db_->NewIterator(read_options);
240 db_->CompactRange(compact_options, &first_slice, &last_slice);
241 // 3 sst after compaction with live iterator
242 CheckFileTypeCounts(dbname_, 0, 3, 1);
243 test::SleepingBackgroundTask sleeping_task_before;
244 env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
245 &sleeping_task_before, Env::Priority::HIGH);
246 delete itr;
247 test::SleepingBackgroundTask sleeping_task_after;
248 env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
249 &sleeping_task_after, Env::Priority::HIGH);
250
251 // Make sure no purges are executed foreground
252 CheckFileTypeCounts(dbname_, 0, 3, 1);
253 sleeping_task_before.WakeUp();
254 sleeping_task_before.WaitUntilDone();
255
256 // Make sure all background purges are executed
257 sleeping_task_after.WakeUp();
258 sleeping_task_after.WaitUntilDone();
259 // 1 sst after iterator deletion
260 CheckFileTypeCounts(dbname_, 0, 1, 1);
261 }
262
TEST_F(DeleteFileTest,BackgroundPurgeCFDropTest)263 TEST_F(DeleteFileTest, BackgroundPurgeCFDropTest) {
264 Options options = CurrentOptions();
265 SetOptions(&options);
266 Destroy(options);
267 options.create_if_missing = true;
268 Reopen(options);
269
270 auto do_test = [&](bool bg_purge) {
271 ColumnFamilyOptions co;
272 co.max_write_buffer_size_to_maintain =
273 static_cast<int64_t>(co.write_buffer_size);
274 WriteOptions wo;
275 FlushOptions fo;
276 ColumnFamilyHandle* cfh = nullptr;
277
278 ASSERT_OK(db_->CreateColumnFamily(co, "dropme", &cfh));
279
280 ASSERT_OK(db_->Put(wo, cfh, "pika", "chu"));
281 ASSERT_OK(db_->Flush(fo, cfh));
282 // Expect 1 sst file.
283 CheckFileTypeCounts(dbname_, 0, 1, 1);
284
285 ASSERT_OK(db_->DropColumnFamily(cfh));
286 // Still 1 file, it won't be deleted while ColumnFamilyHandle is alive.
287 CheckFileTypeCounts(dbname_, 0, 1, 1);
288
289 delete cfh;
290 test::SleepingBackgroundTask sleeping_task_after;
291 env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
292 &sleeping_task_after, Env::Priority::HIGH);
293 // If background purge is enabled, the file should still be there.
294 CheckFileTypeCounts(dbname_, 0, bg_purge ? 1 : 0, 1);
295 TEST_SYNC_POINT("DeleteFileTest::BackgroundPurgeCFDropTest:1");
296
297 // Execute background purges.
298 sleeping_task_after.WakeUp();
299 sleeping_task_after.WaitUntilDone();
300 // The file should have been deleted.
301 CheckFileTypeCounts(dbname_, 0, 0, 1);
302 };
303
304 {
305 SCOPED_TRACE("avoid_unnecessary_blocking_io = false");
306 do_test(false);
307 }
308
309 SyncPoint::GetInstance()->DisableProcessing();
310 SyncPoint::GetInstance()->ClearAllCallBacks();
311 SyncPoint::GetInstance()->LoadDependency(
312 {{"DeleteFileTest::BackgroundPurgeCFDropTest:1",
313 "DBImpl::BGWorkPurge:start"}});
314 SyncPoint::GetInstance()->EnableProcessing();
315
316 options.avoid_unnecessary_blocking_io = true;
317 options.create_if_missing = false;
318 Reopen(options);
319 {
320 SCOPED_TRACE("avoid_unnecessary_blocking_io = true");
321 do_test(true);
322 }
323 }
324
325 // This test is to reproduce a bug that read invalid ReadOption in iterator
326 // cleanup function
TEST_F(DeleteFileTest,BackgroundPurgeCopyOptions)327 TEST_F(DeleteFileTest, BackgroundPurgeCopyOptions) {
328 Options options = CurrentOptions();
329 SetOptions(&options);
330 Destroy(options);
331 options.create_if_missing = true;
332 Reopen(options);
333
334 std::string first("0"), last("999999");
335 CompactRangeOptions compact_options;
336 compact_options.change_level = true;
337 compact_options.target_level = 2;
338 Slice first_slice(first), last_slice(last);
339
340 // We keep an iterator alive
341 Iterator* itr = nullptr;
342 CreateTwoLevels();
343 {
344 ReadOptions read_options;
345 read_options.background_purge_on_iterator_cleanup = true;
346 itr = db_->NewIterator(read_options);
347 // ReadOptions is deleted, but iterator cleanup function should not be
348 // affected
349 }
350
351 db_->CompactRange(compact_options, &first_slice, &last_slice);
352 // 3 sst after compaction with live iterator
353 CheckFileTypeCounts(dbname_, 0, 3, 1);
354 delete itr;
355
356 test::SleepingBackgroundTask sleeping_task_after;
357 env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
358 &sleeping_task_after, Env::Priority::HIGH);
359
360 // Make sure all background purges are executed
361 sleeping_task_after.WakeUp();
362 sleeping_task_after.WaitUntilDone();
363 // 1 sst after iterator deletion
364 CheckFileTypeCounts(dbname_, 0, 1, 1);
365 }
366
TEST_F(DeleteFileTest,BackgroundPurgeTestMultipleJobs)367 TEST_F(DeleteFileTest, BackgroundPurgeTestMultipleJobs) {
368 Options options = CurrentOptions();
369 SetOptions(&options);
370 Destroy(options);
371 options.create_if_missing = true;
372 Reopen(options);
373
374 std::string first("0"), last("999999");
375 CompactRangeOptions compact_options;
376 compact_options.change_level = true;
377 compact_options.target_level = 2;
378 Slice first_slice(first), last_slice(last);
379
380 // We keep an iterator alive
381 CreateTwoLevels();
382 ReadOptions read_options;
383 read_options.background_purge_on_iterator_cleanup = true;
384 Iterator* itr1 = db_->NewIterator(read_options);
385 CreateTwoLevels();
386 Iterator* itr2 = db_->NewIterator(read_options);
387 db_->CompactRange(compact_options, &first_slice, &last_slice);
388 // 5 sst files after 2 compactions with 2 live iterators
389 CheckFileTypeCounts(dbname_, 0, 5, 1);
390
391 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
392 // ~DBImpl should wait until all BGWorkPurge are finished
393 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
394 {{"DBImpl::~DBImpl:WaitJob", "DBImpl::BGWorkPurge"},
395 {"DeleteFileTest::GuardFinish",
396 "DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose"}});
397 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
398
399 delete itr1;
400 env_->Schedule(&DeleteFileTest::DoSleep, this, Env::Priority::HIGH);
401 delete itr2;
402 env_->Schedule(&DeleteFileTest::GuardFinish, nullptr, Env::Priority::HIGH);
403 Close();
404
405 TEST_SYNC_POINT("DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose");
406 // 1 sst after iterator deletion
407 CheckFileTypeCounts(dbname_, 0, 1, 1);
408 }
409
TEST_F(DeleteFileTest,DeleteFileWithIterator)410 TEST_F(DeleteFileTest, DeleteFileWithIterator) {
411 Options options = CurrentOptions();
412 SetOptions(&options);
413 Destroy(options);
414 options.create_if_missing = true;
415 Reopen(options);
416
417 CreateTwoLevels();
418 ReadOptions read_options;
419 Iterator* it = db_->NewIterator(read_options);
420 std::vector<LiveFileMetaData> metadata;
421 db_->GetLiveFilesMetaData(&metadata);
422
423 std::string level2file;
424
425 ASSERT_EQ(metadata.size(), static_cast<size_t>(2));
426 if (metadata[0].level == 1) {
427 level2file = metadata[1].name;
428 } else {
429 level2file = metadata[0].name;
430 }
431
432 Status status = db_->DeleteFile(level2file);
433 fprintf(stdout, "Deletion status %s: %s\n",
434 level2file.c_str(), status.ToString().c_str());
435 ASSERT_TRUE(status.ok());
436 it->SeekToFirst();
437 int numKeysIterated = 0;
438 while(it->Valid()) {
439 numKeysIterated++;
440 it->Next();
441 }
442 ASSERT_EQ(numKeysIterated, 50000);
443 delete it;
444 }
445
TEST_F(DeleteFileTest,DeleteLogFiles)446 TEST_F(DeleteFileTest, DeleteLogFiles) {
447 Options options = CurrentOptions();
448 SetOptions(&options);
449 Destroy(options);
450 options.create_if_missing = true;
451 Reopen(options);
452
453 AddKeys(10, 0);
454 VectorLogPtr logfiles;
455 db_->GetSortedWalFiles(logfiles);
456 ASSERT_GT(logfiles.size(), 0UL);
457 // Take the last log file which is expected to be alive and try to delete it
458 // Should not succeed because live logs are not allowed to be deleted
459 std::unique_ptr<LogFile> alive_log = std::move(logfiles.back());
460 ASSERT_EQ(alive_log->Type(), kAliveLogFile);
461 ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName()));
462 fprintf(stdout, "Deleting alive log file %s\n",
463 alive_log->PathName().c_str());
464 ASSERT_TRUE(!db_->DeleteFile(alive_log->PathName()).ok());
465 ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName()));
466 logfiles.clear();
467
468 // Call Flush to bring about a new working log file and add more keys
469 // Call Flush again to flush out memtable and move alive log to archived log
470 // and try to delete the archived log file
471 FlushOptions fopts;
472 db_->Flush(fopts);
473 AddKeys(10, 0);
474 db_->Flush(fopts);
475 db_->GetSortedWalFiles(logfiles);
476 ASSERT_GT(logfiles.size(), 0UL);
477 std::unique_ptr<LogFile> archived_log = std::move(logfiles.front());
478 ASSERT_EQ(archived_log->Type(), kArchivedLogFile);
479 ASSERT_OK(env_->FileExists(wal_dir_ + "/" + archived_log->PathName()));
480 fprintf(stdout, "Deleting archived log file %s\n",
481 archived_log->PathName().c_str());
482 ASSERT_OK(db_->DeleteFile(archived_log->PathName()));
483 ASSERT_EQ(Status::NotFound(),
484 env_->FileExists(wal_dir_ + "/" + archived_log->PathName()));
485 }
486
TEST_F(DeleteFileTest,DeleteNonDefaultColumnFamily)487 TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) {
488 Options options = CurrentOptions();
489 SetOptions(&options);
490 Destroy(options);
491 options.create_if_missing = true;
492 Reopen(options);
493 CreateAndReopenWithCF({"new_cf"}, options);
494
495 Random rnd(5);
496 for (int i = 0; i < 1000; ++i) {
497 ASSERT_OK(db_->Put(WriteOptions(), handles_[1], test::RandomKey(&rnd, 10),
498 test::RandomKey(&rnd, 10)));
499 }
500 ASSERT_OK(db_->Flush(FlushOptions(), handles_[1]));
501 for (int i = 0; i < 1000; ++i) {
502 ASSERT_OK(db_->Put(WriteOptions(), handles_[1], test::RandomKey(&rnd, 10),
503 test::RandomKey(&rnd, 10)));
504 }
505 ASSERT_OK(db_->Flush(FlushOptions(), handles_[1]));
506
507 std::vector<LiveFileMetaData> metadata;
508 db_->GetLiveFilesMetaData(&metadata);
509 ASSERT_EQ(2U, metadata.size());
510 ASSERT_EQ("new_cf", metadata[0].column_family_name);
511 ASSERT_EQ("new_cf", metadata[1].column_family_name);
512 auto old_file = metadata[0].smallest_seqno < metadata[1].smallest_seqno
513 ? metadata[0].name
514 : metadata[1].name;
515 auto new_file = metadata[0].smallest_seqno > metadata[1].smallest_seqno
516 ? metadata[0].name
517 : metadata[1].name;
518 ASSERT_TRUE(db_->DeleteFile(new_file).IsInvalidArgument());
519 ASSERT_OK(db_->DeleteFile(old_file));
520
521 {
522 std::unique_ptr<Iterator> itr(db_->NewIterator(ReadOptions(), handles_[1]));
523 int count = 0;
524 for (itr->SeekToFirst(); itr->Valid(); itr->Next()) {
525 ASSERT_OK(itr->status());
526 ++count;
527 }
528 ASSERT_EQ(count, 1000);
529 }
530
531 Close();
532 ReopenWithColumnFamilies({kDefaultColumnFamilyName, "new_cf"}, options);
533
534 {
535 std::unique_ptr<Iterator> itr(db_->NewIterator(ReadOptions(), handles_[1]));
536 int count = 0;
537 for (itr->SeekToFirst(); itr->Valid(); itr->Next()) {
538 ASSERT_OK(itr->status());
539 ++count;
540 }
541 ASSERT_EQ(count, 1000);
542 }
543 }
544
545 } // namespace ROCKSDB_NAMESPACE
546
547 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
548 extern "C" {
549 void RegisterCustomObjects(int argc, char** argv);
550 }
551 #else
RegisterCustomObjects(int,char **)552 void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
553 #endif // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
554
main(int argc,char ** argv)555 int main(int argc, char** argv) {
556 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
557 ::testing::InitGoogleTest(&argc, argv);
558 RegisterCustomObjects(argc, argv);
559 return RUN_ALL_TESTS();
560 }
561
562 #else
563 #include <stdio.h>
564
main(int,char **)565 int main(int /*argc*/, char** /*argv*/) {
566 fprintf(stderr,
567 "SKIPPED as DBImpl::DeleteFile is not supported in ROCKSDB_LITE\n");
568 return 0;
569 }
570
571 #endif // !ROCKSDB_LITE
572