1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #include <cstring>
11
12 #include "options/options_helper.h"
13 #include "rocksdb/convenience.h"
14 #include "test_util/testharness.h"
15
16 #ifndef GFLAGS
17 bool FLAGS_enable_print = false;
18 #else
19 #include "util/gflags_compat.h"
20 using GFLAGS_NAMESPACE::ParseCommandLineFlags;
21 DEFINE_bool(enable_print, false, "Print options generated to console.");
22 #endif // GFLAGS
23
24 namespace ROCKSDB_NAMESPACE {
25
26 // Verify options are settable from options strings.
27 // We take the approach that depends on compiler behavior that copy constructor
28 // won't touch implicit padding bytes, so that the test is fragile.
29 // As a result, we only run the tests to verify new fields in options are
30 // settable through string on limited platforms as it depends on behavior of
31 // compilers.
32 #ifndef ROCKSDB_LITE
33 #if defined OS_LINUX || defined OS_WIN
34 #ifndef __clang__
35
36 class OptionsSettableTest : public testing::Test {
37 public:
OptionsSettableTest()38 OptionsSettableTest() {}
39 };
40
41 const char kSpecialChar = 'z';
42 typedef std::vector<std::pair<size_t, size_t>> OffsetGap;
43
FillWithSpecialChar(char * start_ptr,size_t total_size,const OffsetGap & blacklist)44 void FillWithSpecialChar(char* start_ptr, size_t total_size,
45 const OffsetGap& blacklist) {
46 size_t offset = 0;
47 for (auto& pair : blacklist) {
48 std::memset(start_ptr + offset, kSpecialChar, pair.first - offset);
49 offset = pair.first + pair.second;
50 }
51 std::memset(start_ptr + offset, kSpecialChar, total_size - offset);
52 }
53
NumUnsetBytes(char * start_ptr,size_t total_size,const OffsetGap & blacklist)54 int NumUnsetBytes(char* start_ptr, size_t total_size,
55 const OffsetGap& blacklist) {
56 int total_unset_bytes_base = 0;
57 size_t offset = 0;
58 for (auto& pair : blacklist) {
59 for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) {
60 if (*ptr == kSpecialChar) {
61 total_unset_bytes_base++;
62 }
63 }
64 offset = pair.first + pair.second;
65 }
66 for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) {
67 if (*ptr == kSpecialChar) {
68 total_unset_bytes_base++;
69 }
70 }
71 return total_unset_bytes_base;
72 }
73
74 // If the test fails, likely a new option is added to BlockBasedTableOptions
75 // but it cannot be set through GetBlockBasedTableOptionsFromString(), or the
76 // test is not updated accordingly.
77 // After adding an option, we need to make sure it is settable by
78 // GetBlockBasedTableOptionsFromString() and add the option to the input string
79 // passed to the GetBlockBasedTableOptionsFromString() in this test.
80 // If it is a complicated type, you also need to add the field to
81 // kBbtoBlacklist, and maybe add customized verification for it.
TEST_F(OptionsSettableTest,BlockBasedTableOptionsAllFieldsSettable)82 TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) {
83 // Items in the form of <offset, size>. Need to be in ascending order
84 // and not overlapping. Need to updated if new pointer-option is added.
85 const OffsetGap kBbtoBlacklist = {
86 {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
87 sizeof(std::shared_ptr<FlushBlockPolicyFactory>)},
88 {offsetof(struct BlockBasedTableOptions, block_cache),
89 sizeof(std::shared_ptr<Cache>)},
90 {offsetof(struct BlockBasedTableOptions, persistent_cache),
91 sizeof(std::shared_ptr<PersistentCache>)},
92 {offsetof(struct BlockBasedTableOptions, block_cache_compressed),
93 sizeof(std::shared_ptr<Cache>)},
94 {offsetof(struct BlockBasedTableOptions, filter_policy),
95 sizeof(std::shared_ptr<const FilterPolicy>)},
96 };
97
98 // In this test, we catch a new option of BlockBasedTableOptions that is not
99 // settable through GetBlockBasedTableOptionsFromString().
100 // We count padding bytes of the option struct, and assert it to be the same
101 // as unset bytes of an option struct initialized by
102 // GetBlockBasedTableOptionsFromString().
103
104 char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
105
106 // Count padding bytes by setting all bytes in the memory to a special char,
107 // copy a well constructed struct to this memory and see how many special
108 // bytes left.
109 BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions();
110 FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
111 // It based on the behavior of compiler that padding bytes are not changed
112 // when copying the struct. It's prone to failure when compiler behavior
113 // changes. We verify there is unset bytes to detect the case.
114 *bbto = BlockBasedTableOptions();
115 int unset_bytes_base =
116 NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
117 ASSERT_GT(unset_bytes_base, 0);
118 bbto->~BlockBasedTableOptions();
119
120 // Construct the base option passed into
121 // GetBlockBasedTableOptionsFromString().
122 bbto = new (bbto_ptr) BlockBasedTableOptions();
123 FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
124 // This option is not setable:
125 bbto->use_delta_encoding = true;
126
127 char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
128 BlockBasedTableOptions* new_bbto =
129 new (new_bbto_ptr) BlockBasedTableOptions();
130 FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions),
131 kBbtoBlacklist);
132
133 // Need to update the option string if a new option is added.
134 ASSERT_OK(GetBlockBasedTableOptionsFromString(
135 *bbto,
136 "cache_index_and_filter_blocks=1;"
137 "cache_index_and_filter_blocks_with_high_priority=true;"
138 "pin_l0_filter_and_index_blocks_in_cache=1;"
139 "pin_top_level_index_and_filter=1;"
140 "index_type=kHashSearch;"
141 "data_block_index_type=kDataBlockBinaryAndHash;"
142 "index_shortening=kNoShortening;"
143 "data_block_hash_table_util_ratio=0.75;"
144 "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
145 "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
146 "block_size_deviation=8;block_restart_interval=4; "
147 "metadata_block_size=1024;"
148 "partition_filters=false;"
149 "index_block_restart_interval=4;"
150 "filter_policy=bloomfilter:4:true;whole_key_filtering=1;"
151 "format_version=1;"
152 "hash_index_allow_collision=false;"
153 "verify_compression=true;read_amp_bytes_per_bit=0;"
154 "enable_index_compression=false;"
155 "block_align=true",
156 new_bbto));
157
158 ASSERT_EQ(unset_bytes_base,
159 NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions),
160 kBbtoBlacklist));
161
162 ASSERT_TRUE(new_bbto->block_cache.get() != nullptr);
163 ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr);
164 ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr);
165
166 bbto->~BlockBasedTableOptions();
167 new_bbto->~BlockBasedTableOptions();
168
169 delete[] bbto_ptr;
170 delete[] new_bbto_ptr;
171 }
172
173 // If the test fails, likely a new option is added to DBOptions
174 // but it cannot be set through GetDBOptionsFromString(), or the test is not
175 // updated accordingly.
176 // After adding an option, we need to make sure it is settable by
177 // GetDBOptionsFromString() and add the option to the input string passed to
178 // DBOptionsFromString()in this test.
179 // If it is a complicated type, you also need to add the field to
180 // kDBOptionsBlacklist, and maybe add customized verification for it.
TEST_F(OptionsSettableTest,DBOptionsAllFieldsSettable)181 TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
182 const OffsetGap kDBOptionsBlacklist = {
183 {offsetof(struct DBOptions, env), sizeof(Env*)},
184 {offsetof(struct DBOptions, rate_limiter),
185 sizeof(std::shared_ptr<RateLimiter>)},
186 {offsetof(struct DBOptions, sst_file_manager),
187 sizeof(std::shared_ptr<SstFileManager>)},
188 {offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr<Logger>)},
189 {offsetof(struct DBOptions, statistics),
190 sizeof(std::shared_ptr<Statistics>)},
191 {offsetof(struct DBOptions, db_paths), sizeof(std::vector<DbPath>)},
192 {offsetof(struct DBOptions, db_log_dir), sizeof(std::string)},
193 {offsetof(struct DBOptions, wal_dir), sizeof(std::string)},
194 {offsetof(struct DBOptions, write_buffer_manager),
195 sizeof(std::shared_ptr<WriteBufferManager>)},
196 {offsetof(struct DBOptions, listeners),
197 sizeof(std::vector<std::shared_ptr<EventListener>>)},
198 {offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr<Cache>)},
199 {offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)},
200 {offsetof(struct DBOptions, file_checksum_gen_factory),
201 sizeof(std::shared_ptr<FileChecksumGenFactory>)},
202 };
203
204 char* options_ptr = new char[sizeof(DBOptions)];
205
206 // Count padding bytes by setting all bytes in the memory to a special char,
207 // copy a well constructed struct to this memory and see how many special
208 // bytes left.
209 DBOptions* options = new (options_ptr) DBOptions();
210 FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
211 // It based on the behavior of compiler that padding bytes are not changed
212 // when copying the struct. It's prone to failure when compiler behavior
213 // changes. We verify there is unset bytes to detect the case.
214 *options = DBOptions();
215 int unset_bytes_base =
216 NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
217 ASSERT_GT(unset_bytes_base, 0);
218 options->~DBOptions();
219
220 options = new (options_ptr) DBOptions();
221 FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
222
223 char* new_options_ptr = new char[sizeof(DBOptions)];
224 DBOptions* new_options = new (new_options_ptr) DBOptions();
225 FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
226
227 // Need to update the option string if a new option is added.
228 ASSERT_OK(
229 GetDBOptionsFromString(*options,
230 "wal_bytes_per_sync=4295048118;"
231 "delete_obsolete_files_period_micros=4294967758;"
232 "WAL_ttl_seconds=4295008036;"
233 "WAL_size_limit_MB=4295036161;"
234 "max_write_batch_group_size_bytes=1048576;"
235 "wal_dir=path/to/wal_dir;"
236 "db_write_buffer_size=2587;"
237 "max_subcompactions=64330;"
238 "table_cache_numshardbits=28;"
239 "max_open_files=72;"
240 "max_file_opening_threads=35;"
241 "max_background_jobs=8;"
242 "base_background_compactions=3;"
243 "max_background_compactions=33;"
244 "use_fsync=true;"
245 "use_adaptive_mutex=false;"
246 "max_total_wal_size=4295005604;"
247 "compaction_readahead_size=0;"
248 "new_table_reader_for_compaction_inputs=false;"
249 "keep_log_file_num=4890;"
250 "skip_stats_update_on_db_open=false;"
251 "skip_checking_sst_file_sizes_on_db_open=false;"
252 "max_manifest_file_size=4295009941;"
253 "db_log_dir=path/to/db_log_dir;"
254 "skip_log_error_on_recovery=true;"
255 "writable_file_max_buffer_size=1048576;"
256 "paranoid_checks=true;"
257 "is_fd_close_on_exec=false;"
258 "bytes_per_sync=4295013613;"
259 "strict_bytes_per_sync=true;"
260 "enable_thread_tracking=false;"
261 "recycle_log_file_num=0;"
262 "create_missing_column_families=true;"
263 "log_file_time_to_roll=3097;"
264 "max_background_flushes=35;"
265 "create_if_missing=false;"
266 "error_if_exists=true;"
267 "delayed_write_rate=4294976214;"
268 "manifest_preallocation_size=1222;"
269 "allow_mmap_writes=false;"
270 "stats_dump_period_sec=70127;"
271 "stats_persist_period_sec=54321;"
272 "persist_stats_to_disk=true;"
273 "stats_history_buffer_size=14159;"
274 "allow_fallocate=true;"
275 "allow_mmap_reads=false;"
276 "use_direct_reads=false;"
277 "use_direct_io_for_flush_and_compaction=false;"
278 "max_log_file_size=4607;"
279 "random_access_max_buffer_size=1048576;"
280 "advise_random_on_open=true;"
281 "fail_if_options_file_error=false;"
282 "enable_pipelined_write=false;"
283 "unordered_write=false;"
284 "allow_concurrent_memtable_write=true;"
285 "wal_recovery_mode=kPointInTimeRecovery;"
286 "enable_write_thread_adaptive_yield=true;"
287 "write_thread_slow_yield_usec=5;"
288 "write_thread_max_yield_usec=1000;"
289 "access_hint_on_compaction_start=NONE;"
290 "info_log_level=DEBUG_LEVEL;"
291 "dump_malloc_stats=false;"
292 "allow_2pc=false;"
293 "avoid_flush_during_recovery=false;"
294 "avoid_flush_during_shutdown=false;"
295 "allow_ingest_behind=false;"
296 "preserve_deletes=false;"
297 "concurrent_prepare=false;"
298 "two_write_queues=false;"
299 "manual_wal_flush=false;"
300 "seq_per_batch=false;"
301 "atomic_flush=false;"
302 "avoid_unnecessary_blocking_io=false;"
303 "log_readahead_size=0;"
304 "write_dbid_to_manifest=false;"
305 "best_efforts_recovery=false",
306 new_options));
307
308 ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),
309 kDBOptionsBlacklist));
310
311 options->~DBOptions();
312 new_options->~DBOptions();
313
314 delete[] options_ptr;
315 delete[] new_options_ptr;
316 }
317
318 template <typename T1, typename T2>
offset_of(T1 T2::* member)319 inline int offset_of(T1 T2::*member) {
320 static T2 obj;
321 return int(size_t(&(obj.*member)) - size_t(&obj));
322 }
323
324 // If the test fails, likely a new option is added to ColumnFamilyOptions
325 // but it cannot be set through GetColumnFamilyOptionsFromString(), or the
326 // test is not updated accordingly.
327 // After adding an option, we need to make sure it is settable by
328 // GetColumnFamilyOptionsFromString() and add the option to the input
329 // string passed to GetColumnFamilyOptionsFromString()in this test.
330 // If it is a complicated type, you also need to add the field to
331 // kColumnFamilyOptionsBlacklist, and maybe add customized verification
332 // for it.
TEST_F(OptionsSettableTest,ColumnFamilyOptionsAllFieldsSettable)333 TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
334 // options in the blacklist need to appear in the same order as in
335 // ColumnFamilyOptions.
336 const OffsetGap kColumnFamilyOptionsBlacklist = {
337 {offset_of(&ColumnFamilyOptions::inplace_callback),
338 sizeof(UpdateStatus(*)(char*, uint32_t*, Slice, std::string*))},
339 {offset_of(
340 &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor),
341 sizeof(std::shared_ptr<const SliceTransform>)},
342 {offset_of(&ColumnFamilyOptions::compression_per_level),
343 sizeof(std::vector<CompressionType>)},
344 {offset_of(
345 &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional),
346 sizeof(std::vector<int>)},
347 {offset_of(&ColumnFamilyOptions::memtable_factory),
348 sizeof(std::shared_ptr<MemTableRepFactory>)},
349 {offset_of(&ColumnFamilyOptions::table_properties_collector_factories),
350 sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)},
351 {offset_of(&ColumnFamilyOptions::comparator), sizeof(Comparator*)},
352 {offset_of(&ColumnFamilyOptions::merge_operator),
353 sizeof(std::shared_ptr<MergeOperator>)},
354 {offset_of(&ColumnFamilyOptions::compaction_filter),
355 sizeof(const CompactionFilter*)},
356 {offset_of(&ColumnFamilyOptions::compaction_filter_factory),
357 sizeof(std::shared_ptr<CompactionFilterFactory>)},
358 {offset_of(&ColumnFamilyOptions::prefix_extractor),
359 sizeof(std::shared_ptr<const SliceTransform>)},
360 {offset_of(&ColumnFamilyOptions::snap_refresh_nanos), sizeof(uint64_t)},
361 {offset_of(&ColumnFamilyOptions::table_factory),
362 sizeof(std::shared_ptr<TableFactory>)},
363 {offset_of(&ColumnFamilyOptions::cf_paths), sizeof(std::vector<DbPath>)},
364 {offset_of(&ColumnFamilyOptions::compaction_thread_limiter),
365 sizeof(std::shared_ptr<ConcurrentTaskLimiter>)},
366 };
367
368 char* options_ptr = new char[sizeof(ColumnFamilyOptions)];
369
370 // Count padding bytes by setting all bytes in the memory to a special char,
371 // copy a well constructed struct to this memory and see how many special
372 // bytes left.
373 ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions();
374 FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
375 kColumnFamilyOptionsBlacklist);
376 // It based on the behavior of compiler that padding bytes are not changed
377 // when copying the struct. It's prone to failure when compiler behavior
378 // changes. We verify there is unset bytes to detect the case.
379 *options = ColumnFamilyOptions();
380
381 // Deprecatd option which is not initialized. Need to set it to avoid
382 // Valgrind error
383 options->max_mem_compaction_level = 0;
384
385 int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions),
386 kColumnFamilyOptionsBlacklist);
387 ASSERT_GT(unset_bytes_base, 0);
388 options->~ColumnFamilyOptions();
389
390 options = new (options_ptr) ColumnFamilyOptions();
391 FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
392 kColumnFamilyOptionsBlacklist);
393
394 // Following options are not settable through
395 // GetColumnFamilyOptionsFromString():
396 options->rate_limit_delay_max_milliseconds = 33;
397 options->compaction_options_universal = CompactionOptionsUniversal();
398 options->compression_opts = CompressionOptions();
399 options->bottommost_compression_opts = CompressionOptions();
400 options->hard_rate_limit = 0;
401 options->soft_rate_limit = 0;
402 options->purge_redundant_kvs_while_flush = false;
403 options->max_mem_compaction_level = 0;
404 options->compaction_filter = nullptr;
405
406 char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)];
407 ColumnFamilyOptions* new_options =
408 new (new_options_ptr) ColumnFamilyOptions();
409 FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions),
410 kColumnFamilyOptionsBlacklist);
411
412 // Need to update the option string if a new option is added.
413 ASSERT_OK(GetColumnFamilyOptionsFromString(
414 *options,
415 "compaction_filter_factory=mpudlojcujCompactionFilterFactory;"
416 "table_factory=PlainTable;"
417 "prefix_extractor=rocksdb.CappedPrefix.13;"
418 "comparator=leveldb.BytewiseComparator;"
419 "compression_per_level=kBZip2Compression:kBZip2Compression:"
420 "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:"
421 "kSnappyCompression;"
422 "max_bytes_for_level_base=986;"
423 "bloom_locality=8016;"
424 "target_file_size_base=4294976376;"
425 "memtable_huge_page_size=2557;"
426 "max_successive_merges=5497;"
427 "max_sequential_skip_in_iterations=4294971408;"
428 "arena_block_size=1893;"
429 "target_file_size_multiplier=35;"
430 "min_write_buffer_number_to_merge=9;"
431 "max_write_buffer_number=84;"
432 "write_buffer_size=1653;"
433 "max_compaction_bytes=64;"
434 "max_bytes_for_level_multiplier=60;"
435 "memtable_factory=SkipListFactory;"
436 "compression=kNoCompression;"
437 "bottommost_compression=kDisableCompressionOption;"
438 "level0_stop_writes_trigger=33;"
439 "num_levels=99;"
440 "level0_slowdown_writes_trigger=22;"
441 "level0_file_num_compaction_trigger=14;"
442 "compaction_filter=urxcqstuwnCompactionFilter;"
443 "soft_rate_limit=530.615385;"
444 "soft_pending_compaction_bytes_limit=0;"
445 "max_write_buffer_number_to_maintain=84;"
446 "max_write_buffer_size_to_maintain=2147483648;"
447 "merge_operator=aabcxehazrMergeOperator;"
448 "memtable_prefix_bloom_size_ratio=0.4642;"
449 "memtable_whole_key_filtering=true;"
450 "memtable_insert_with_hint_prefix_extractor=rocksdb.CappedPrefix.13;"
451 "paranoid_file_checks=true;"
452 "force_consistency_checks=true;"
453 "inplace_update_num_locks=7429;"
454 "optimize_filters_for_hits=false;"
455 "level_compaction_dynamic_level_bytes=false;"
456 "inplace_update_support=false;"
457 "compaction_style=kCompactionStyleFIFO;"
458 "compaction_pri=kMinOverlappingRatio;"
459 "hard_pending_compaction_bytes_limit=0;"
460 "disable_auto_compactions=false;"
461 "report_bg_io_stats=true;"
462 "ttl=60;"
463 "periodic_compaction_seconds=3600;"
464 "sample_for_compression=0;"
465 "compaction_options_fifo={max_table_files_size=3;allow_"
466 "compaction=false;};",
467 new_options));
468
469 ASSERT_EQ(unset_bytes_base,
470 NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions),
471 kColumnFamilyOptionsBlacklist));
472
473 options->~ColumnFamilyOptions();
474 new_options->~ColumnFamilyOptions();
475
476 delete[] options_ptr;
477 delete[] new_options_ptr;
478 }
479 #endif // !__clang__
480 #endif // OS_LINUX || OS_WIN
481 #endif // !ROCKSDB_LITE
482
483 } // namespace ROCKSDB_NAMESPACE
484
main(int argc,char ** argv)485 int main(int argc, char** argv) {
486 ::testing::InitGoogleTest(&argc, argv);
487 #ifdef GFLAGS
488 ParseCommandLineFlags(&argc, &argv, true);
489 #endif // GFLAGS
490 return RUN_ALL_TESTS();
491 }
492