1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2 //  This source code is licensed under both the GPLv2 (found in the
3 //  COPYING file in the root directory) and Apache 2.0 License
4 //  (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9 
10 // Introduction of SyncPoint effectively disabled building and running this test
11 // in Release build.
12 // which is a pity, it is a good test
13 #if !defined(ROCKSDB_LITE)
14 
15 #include "db/db_test_util.h"
16 #include "port/port.h"
17 #include "port/stack_trace.h"
18 
19 namespace ROCKSDB_NAMESPACE {
20 class DBTestDynamicLevel : public DBTestBase {
21  public:
DBTestDynamicLevel()22   DBTestDynamicLevel() : DBTestBase("/db_dynamic_level_test") {}
23 };
24 
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesBase)25 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
26   if (!Snappy_Supported() || !LZ4_Supported()) {
27     return;
28   }
29   // Use InMemoryEnv, or it would be too slow.
30   std::unique_ptr<Env> env(new MockEnv(env_));
31 
32   const int kNKeys = 1000;
33   int keys[kNKeys];
34 
35   auto verify_func = [&]() {
36     for (int i = 0; i < kNKeys; i++) {
37       ASSERT_NE("NOT_FOUND", Get(Key(i)));
38       ASSERT_NE("NOT_FOUND", Get(Key(kNKeys * 2 + i)));
39       if (i < kNKeys / 10) {
40         ASSERT_EQ("NOT_FOUND", Get(Key(kNKeys + keys[i])));
41       } else {
42         ASSERT_NE("NOT_FOUND", Get(Key(kNKeys + keys[i])));
43       }
44     }
45   };
46 
47   Random rnd(301);
48   for (int ordered_insert = 0; ordered_insert <= 1; ordered_insert++) {
49     for (int i = 0; i < kNKeys; i++) {
50       keys[i] = i;
51     }
52     if (ordered_insert == 0) {
53       std::random_shuffle(std::begin(keys), std::end(keys));
54     }
55     for (int max_background_compactions = 1; max_background_compactions < 4;
56          max_background_compactions += 2) {
57       Options options;
58       options.env = env.get();
59       options.create_if_missing = true;
60       options.write_buffer_size = 2048;
61       options.max_write_buffer_number = 2;
62       options.level0_file_num_compaction_trigger = 2;
63       options.level0_slowdown_writes_trigger = 2;
64       options.level0_stop_writes_trigger = 2;
65       options.target_file_size_base = 2048;
66       options.level_compaction_dynamic_level_bytes = true;
67       options.max_bytes_for_level_base = 10240;
68       options.max_bytes_for_level_multiplier = 4;
69       options.soft_rate_limit = 1.1;
70       options.max_background_compactions = max_background_compactions;
71       options.num_levels = 5;
72 
73       options.compression_per_level.resize(3);
74       options.compression_per_level[0] = kNoCompression;
75       options.compression_per_level[1] = kLZ4Compression;
76       options.compression_per_level[2] = kSnappyCompression;
77       options.env = env_;
78 
79       DestroyAndReopen(options);
80 
81       for (int i = 0; i < kNKeys; i++) {
82         int key = keys[i];
83         ASSERT_OK(Put(Key(kNKeys + key), RandomString(&rnd, 102)));
84         ASSERT_OK(Put(Key(key), RandomString(&rnd, 102)));
85         ASSERT_OK(Put(Key(kNKeys * 2 + key), RandomString(&rnd, 102)));
86         ASSERT_OK(Delete(Key(kNKeys + keys[i / 10])));
87         env_->SleepForMicroseconds(5000);
88       }
89 
90       uint64_t int_prop;
91       ASSERT_TRUE(db_->GetIntProperty("rocksdb.background-errors", &int_prop));
92       ASSERT_EQ(0U, int_prop);
93 
94       // Verify DB
95       for (int j = 0; j < 2; j++) {
96         verify_func();
97         if (j == 0) {
98           Reopen(options);
99         }
100       }
101 
102       // Test compact range works
103       dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
104       // All data should be in the last level.
105       ColumnFamilyMetaData cf_meta;
106       db_->GetColumnFamilyMetaData(&cf_meta);
107       ASSERT_EQ(5U, cf_meta.levels.size());
108       for (int i = 0; i < 4; i++) {
109         ASSERT_EQ(0U, cf_meta.levels[i].files.size());
110       }
111       ASSERT_GT(cf_meta.levels[4U].files.size(), 0U);
112       verify_func();
113 
114       Close();
115     }
116   }
117 
118   env_->SetBackgroundThreads(1, Env::LOW);
119   env_->SetBackgroundThreads(1, Env::HIGH);
120 }
121 
122 // Test specific cases in dynamic max bytes
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesBase2)123 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
124   Random rnd(301);
125   int kMaxKey = 1000000;
126 
127   Options options = CurrentOptions();
128   options.compression = kNoCompression;
129   options.create_if_missing = true;
130   options.write_buffer_size = 20480;
131   options.max_write_buffer_number = 2;
132   options.level0_file_num_compaction_trigger = 2;
133   options.level0_slowdown_writes_trigger = 9999;
134   options.level0_stop_writes_trigger = 9999;
135   options.target_file_size_base = 9102;
136   options.level_compaction_dynamic_level_bytes = true;
137   options.max_bytes_for_level_base = 40960;
138   options.max_bytes_for_level_multiplier = 4;
139   options.max_background_compactions = 2;
140   options.num_levels = 5;
141   options.max_compaction_bytes = 0;  // Force not expanding in compactions
142   BlockBasedTableOptions table_options;
143   table_options.block_size = 1024;
144   options.table_factory.reset(NewBlockBasedTableFactory(table_options));
145 
146   DestroyAndReopen(options);
147   ASSERT_OK(dbfull()->SetOptions({
148       {"disable_auto_compactions", "true"},
149   }));
150 
151   uint64_t int_prop;
152   std::string str_prop;
153 
154   // Initial base level is the last level
155   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
156   ASSERT_EQ(4U, int_prop);
157 
158   // Put about 28K to L0
159   for (int i = 0; i < 70; i++) {
160     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
161                   RandomString(&rnd, 380)));
162   }
163   ASSERT_OK(dbfull()->SetOptions({
164       {"disable_auto_compactions", "false"},
165   }));
166   Flush();
167   dbfull()->TEST_WaitForCompact();
168   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
169   ASSERT_EQ(4U, int_prop);
170 
171   // Insert extra about 28K to L0. After they are compacted to L4, the base
172   // level should be changed to L3.
173   ASSERT_OK(dbfull()->SetOptions({
174       {"disable_auto_compactions", "true"},
175   }));
176   for (int i = 0; i < 70; i++) {
177     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
178                   RandomString(&rnd, 380)));
179   }
180 
181   ASSERT_OK(dbfull()->SetOptions({
182       {"disable_auto_compactions", "false"},
183   }));
184   Flush();
185   dbfull()->TEST_WaitForCompact();
186   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
187   ASSERT_EQ(3U, int_prop);
188   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level1", &str_prop));
189   ASSERT_EQ("0", str_prop);
190   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level2", &str_prop));
191   ASSERT_EQ("0", str_prop);
192 
193   // Write even more data while leaving the base level at L3.
194   ASSERT_OK(dbfull()->SetOptions({
195       {"disable_auto_compactions", "true"},
196   }));
197   // Write about 40K more
198   for (int i = 0; i < 100; i++) {
199     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
200                   RandomString(&rnd, 380)));
201   }
202   ASSERT_OK(dbfull()->SetOptions({
203       {"disable_auto_compactions", "false"},
204   }));
205   Flush();
206   dbfull()->TEST_WaitForCompact();
207   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
208   ASSERT_EQ(3U, int_prop);
209 
210   // Fill up L0, and then run an (auto) L0->Lmax compaction to raise the base
211   // level to 2.
212   ASSERT_OK(dbfull()->SetOptions({
213       {"disable_auto_compactions", "true"},
214   }));
215   // Write about 650K more.
216   // Each file is about 11KB, with 9KB of data.
217   for (int i = 0; i < 1300; i++) {
218     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
219                   RandomString(&rnd, 380)));
220   }
221 
222   // Make sure that the compaction starts before the last bit of data is
223   // flushed, so that the base level isn't raised to L1.
224   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
225       {"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:0"},
226   });
227   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
228 
229   ASSERT_OK(dbfull()->SetOptions({
230       {"disable_auto_compactions", "false"},
231   }));
232 
233   TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:0");
234   Flush();
235   dbfull()->TEST_WaitForCompact();
236   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
237   ASSERT_EQ(2U, int_prop);
238   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
239   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
240 
241   // Write more data until the base level changes to L1. There will be
242   // a manual compaction going on at the same time.
243   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
244       {"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:1"},
245       {"DynamicLevelMaxBytesBase2:2", "CompactionJob::Run():End"},
246       {"DynamicLevelMaxBytesBase2:compact_range_finish",
247        "FlushJob::WriteLevel0Table"},
248   });
249   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
250 
251   ROCKSDB_NAMESPACE::port::Thread thread([this] {
252     TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_start");
253     ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
254     TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_finish");
255   });
256 
257   TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:1");
258   for (int i = 0; i < 2; i++) {
259     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
260                   RandomString(&rnd, 380)));
261   }
262   TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:2");
263 
264   Flush();
265 
266   thread.join();
267 
268   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
269   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
270 
271   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
272   ASSERT_EQ(1U, int_prop);
273 }
274 
275 // Test specific cases in dynamic max bytes
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesCompactRange)276 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesCompactRange) {
277   Random rnd(301);
278   int kMaxKey = 1000000;
279 
280   Options options = CurrentOptions();
281   options.create_if_missing = true;
282   options.write_buffer_size = 2048;
283   options.max_write_buffer_number = 2;
284   options.level0_file_num_compaction_trigger = 2;
285   options.level0_slowdown_writes_trigger = 9999;
286   options.level0_stop_writes_trigger = 9999;
287   options.target_file_size_base = 2;
288   options.level_compaction_dynamic_level_bytes = true;
289   options.max_bytes_for_level_base = 10240;
290   options.max_bytes_for_level_multiplier = 4;
291   options.max_background_compactions = 1;
292   const int kNumLevels = 5;
293   options.num_levels = kNumLevels;
294   options.max_compaction_bytes = 1;  // Force not expanding in compactions
295   BlockBasedTableOptions table_options;
296   table_options.block_size = 1024;
297   options.table_factory.reset(NewBlockBasedTableFactory(table_options));
298 
299   DestroyAndReopen(options);
300 
301   // Compact against empty DB
302   dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
303 
304   uint64_t int_prop;
305   std::string str_prop;
306 
307   // Initial base level is the last level
308   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
309   ASSERT_EQ(4U, int_prop);
310 
311   // Put about 7K to L0
312   for (int i = 0; i < 140; i++) {
313     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
314                   RandomString(&rnd, 80)));
315   }
316   Flush();
317   dbfull()->TEST_WaitForCompact();
318   if (NumTableFilesAtLevel(0) == 0) {
319     // Make sure level 0 is not empty
320     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
321                   RandomString(&rnd, 80)));
322     Flush();
323   }
324 
325   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
326   ASSERT_EQ(3U, int_prop);
327   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level1", &str_prop));
328   ASSERT_EQ("0", str_prop);
329   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level2", &str_prop));
330   ASSERT_EQ("0", str_prop);
331 
332   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
333   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
334 
335   std::set<int> output_levels;
336   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
337       "CompactionPicker::CompactRange:Return", [&](void* arg) {
338         Compaction* compaction = reinterpret_cast<Compaction*>(arg);
339         output_levels.insert(compaction->output_level());
340       });
341   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
342 
343   dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
344   ASSERT_EQ(output_levels.size(), 2);
345   ASSERT_TRUE(output_levels.find(3) != output_levels.end());
346   ASSERT_TRUE(output_levels.find(4) != output_levels.end());
347   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level0", &str_prop));
348   ASSERT_EQ("0", str_prop);
349   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level3", &str_prop));
350   ASSERT_EQ("0", str_prop);
351   // Base level is still level 3.
352   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
353   ASSERT_EQ(3U, int_prop);
354 }
355 
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesBaseInc)356 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
357   Options options = CurrentOptions();
358   options.create_if_missing = true;
359   options.write_buffer_size = 2048;
360   options.max_write_buffer_number = 2;
361   options.level0_file_num_compaction_trigger = 2;
362   options.level0_slowdown_writes_trigger = 2;
363   options.level0_stop_writes_trigger = 2;
364   options.target_file_size_base = 2048;
365   options.level_compaction_dynamic_level_bytes = true;
366   options.max_bytes_for_level_base = 10240;
367   options.max_bytes_for_level_multiplier = 4;
368   options.soft_rate_limit = 1.1;
369   options.max_background_compactions = 2;
370   options.num_levels = 5;
371   options.max_compaction_bytes = 100000000;
372 
373   DestroyAndReopen(options);
374 
375   int non_trivial = 0;
376   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
377       "DBImpl::BackgroundCompaction:NonTrivial",
378       [&](void* /*arg*/) { non_trivial++; });
379   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
380 
381   Random rnd(301);
382   const int total_keys = 3000;
383   const int random_part_size = 100;
384   for (int i = 0; i < total_keys; i++) {
385     std::string value = RandomString(&rnd, random_part_size);
386     PutFixed32(&value, static_cast<uint32_t>(i));
387     ASSERT_OK(Put(Key(i), value));
388   }
389   Flush();
390   dbfull()->TEST_WaitForCompact();
391   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
392 
393   ASSERT_EQ(non_trivial, 0);
394 
395   for (int i = 0; i < total_keys; i++) {
396     std::string value = Get(Key(i));
397     ASSERT_EQ(DecodeFixed32(value.c_str() + random_part_size),
398               static_cast<uint32_t>(i));
399   }
400 
401   env_->SetBackgroundThreads(1, Env::LOW);
402   env_->SetBackgroundThreads(1, Env::HIGH);
403 }
404 
TEST_F(DBTestDynamicLevel,DISABLED_MigrateToDynamicLevelMaxBytesBase)405 TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
406   Random rnd(301);
407   const int kMaxKey = 2000;
408 
409   Options options;
410   options.create_if_missing = true;
411   options.write_buffer_size = 2048;
412   options.max_write_buffer_number = 8;
413   options.level0_file_num_compaction_trigger = 4;
414   options.level0_slowdown_writes_trigger = 4;
415   options.level0_stop_writes_trigger = 8;
416   options.target_file_size_base = 2048;
417   options.level_compaction_dynamic_level_bytes = false;
418   options.max_bytes_for_level_base = 10240;
419   options.max_bytes_for_level_multiplier = 4;
420   options.soft_rate_limit = 1.1;
421   options.num_levels = 8;
422 
423   DestroyAndReopen(options);
424 
425   auto verify_func = [&](int num_keys, bool if_sleep) {
426     for (int i = 0; i < num_keys; i++) {
427       ASSERT_NE("NOT_FOUND", Get(Key(kMaxKey + i)));
428       if (i < num_keys / 10) {
429         ASSERT_EQ("NOT_FOUND", Get(Key(i)));
430       } else {
431         ASSERT_NE("NOT_FOUND", Get(Key(i)));
432       }
433       if (if_sleep && i % 1000 == 0) {
434         // Without it, valgrind may choose not to give another
435         // thread a chance to run before finishing the function,
436         // causing the test to be extremely slow.
437         env_->SleepForMicroseconds(1);
438       }
439     }
440   };
441 
442   int total_keys = 1000;
443   for (int i = 0; i < total_keys; i++) {
444     ASSERT_OK(Put(Key(i), RandomString(&rnd, 102)));
445     ASSERT_OK(Put(Key(kMaxKey + i), RandomString(&rnd, 102)));
446     ASSERT_OK(Delete(Key(i / 10)));
447   }
448   verify_func(total_keys, false);
449   dbfull()->TEST_WaitForCompact();
450 
451   options.level_compaction_dynamic_level_bytes = true;
452   options.disable_auto_compactions = true;
453   Reopen(options);
454   verify_func(total_keys, false);
455 
456   std::atomic_bool compaction_finished;
457   compaction_finished = false;
458   // Issue manual compaction in one thread and still verify DB state
459   // in main thread.
460   ROCKSDB_NAMESPACE::port::Thread t([&]() {
461     CompactRangeOptions compact_options;
462     compact_options.change_level = true;
463     compact_options.target_level = options.num_levels - 1;
464     dbfull()->CompactRange(compact_options, nullptr, nullptr);
465     compaction_finished.store(true);
466   });
467   do {
468     verify_func(total_keys, true);
469   } while (!compaction_finished.load());
470   t.join();
471 
472   ASSERT_OK(dbfull()->SetOptions({
473       {"disable_auto_compactions", "false"},
474   }));
475 
476   int total_keys2 = 2000;
477   for (int i = total_keys; i < total_keys2; i++) {
478     ASSERT_OK(Put(Key(i), RandomString(&rnd, 102)));
479     ASSERT_OK(Put(Key(kMaxKey + i), RandomString(&rnd, 102)));
480     ASSERT_OK(Delete(Key(i / 10)));
481   }
482 
483   verify_func(total_keys2, false);
484   dbfull()->TEST_WaitForCompact();
485   verify_func(total_keys2, false);
486 
487   // Base level is not level 1
488   ASSERT_EQ(NumTableFilesAtLevel(1), 0);
489   ASSERT_EQ(NumTableFilesAtLevel(2), 0);
490 }
491 }  // namespace ROCKSDB_NAMESPACE
492 
493 #endif  // !defined(ROCKSDB_LITE)
494 
main(int argc,char ** argv)495 int main(int argc, char** argv) {
496 #if !defined(ROCKSDB_LITE)
497   ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
498   ::testing::InitGoogleTest(&argc, argv);
499   return RUN_ALL_TESTS();
500 #else
501   (void) argc;
502   (void) argv;
503   return 0;
504 #endif
505 }
506