1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #include "rocksdb/db_bench_tool.h"
11 #include "options/options_parser.h"
12 #include "rocksdb/utilities/options_util.h"
13 #include "test_util/testharness.h"
14 #include "test_util/testutil.h"
15 #include "util/random.h"
16
17 #ifdef GFLAGS
18 #include "util/gflags_compat.h"
19
20 namespace ROCKSDB_NAMESPACE {
21 namespace {
22 static const int kMaxArgCount = 100;
23 static const size_t kArgBufferSize = 100000;
24 } // namespace
25
26 class DBBenchTest : public testing::Test {
27 public:
DBBenchTest()28 DBBenchTest() : rnd_(0xFB) {
29 test_path_ = test::PerThreadDBPath("db_bench_test");
30 Env::Default()->CreateDir(test_path_);
31 db_path_ = test_path_ + "/db";
32 wal_path_ = test_path_ + "/wal";
33 }
34
~DBBenchTest()35 ~DBBenchTest() {
36 // DestroyDB(db_path_, Options());
37 }
38
ResetArgs()39 void ResetArgs() {
40 argc_ = 0;
41 cursor_ = 0;
42 memset(arg_buffer_, 0, kArgBufferSize);
43 }
44
AppendArgs(const std::vector<std::string> & args)45 void AppendArgs(const std::vector<std::string>& args) {
46 for (const auto& arg : args) {
47 ASSERT_LE(cursor_ + arg.size() + 1, kArgBufferSize);
48 ASSERT_LE(argc_ + 1, kMaxArgCount);
49 snprintf(arg_buffer_ + cursor_, arg.size() + 1, "%s", arg.c_str());
50
51 argv_[argc_++] = arg_buffer_ + cursor_;
52 cursor_ += arg.size() + 1;
53 }
54 }
55
RunDbBench(const std::string & options_file_name)56 void RunDbBench(const std::string& options_file_name) {
57 AppendArgs({"./db_bench", "--benchmarks=fillseq", "--use_existing_db=0",
58 "--num=1000",
59 std::string(std::string("--db=") + db_path_).c_str(),
60 std::string(std::string("--wal_dir=") + wal_path_).c_str(),
61 std::string(std::string("--options_file=") + options_file_name)
62 .c_str()});
63 ASSERT_EQ(0, db_bench_tool(argc(), argv()));
64 }
65
VerifyOptions(const Options & opt)66 void VerifyOptions(const Options& opt) {
67 DBOptions loaded_db_opts;
68 std::vector<ColumnFamilyDescriptor> cf_descs;
69 ASSERT_OK(LoadLatestOptions(db_path_, FileSystem::Default(),
70 &loaded_db_opts, &cf_descs));
71
72 ASSERT_OK(
73 RocksDBOptionsParser::VerifyDBOptions(DBOptions(opt), loaded_db_opts));
74 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(opt),
75 cf_descs[0].options));
76
77 // check with the default rocksdb options and expect failure
78 ASSERT_NOK(
79 RocksDBOptionsParser::VerifyDBOptions(DBOptions(), loaded_db_opts));
80 ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(ColumnFamilyOptions(),
81 cf_descs[0].options));
82 }
83
argv()84 char** argv() { return argv_; }
85
argc()86 int argc() { return argc_; }
87
88 std::string db_path_;
89 std::string test_path_;
90 std::string wal_path_;
91
92 char arg_buffer_[kArgBufferSize];
93 char* argv_[kMaxArgCount];
94 int argc_ = 0;
95 int cursor_ = 0;
96 Random rnd_;
97 };
98
99 namespace {} // namespace
100
TEST_F(DBBenchTest,OptionsFile)101 TEST_F(DBBenchTest, OptionsFile) {
102 const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
103
104 Options opt;
105 opt.create_if_missing = true;
106 opt.max_open_files = 256;
107 opt.max_background_compactions = 10;
108 opt.arena_block_size = 8388608;
109 ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
110 {ColumnFamilyOptions(opt)}, kOptionsFileName,
111 Env::Default()));
112
113 // override the following options as db_bench will not take these
114 // options from the options file
115 opt.wal_dir = wal_path_;
116
117 RunDbBench(kOptionsFileName);
118
119 VerifyOptions(opt);
120 }
121
TEST_F(DBBenchTest,OptionsFileUniversal)122 TEST_F(DBBenchTest, OptionsFileUniversal) {
123 const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
124
125 Options opt;
126 opt.compaction_style = kCompactionStyleUniversal;
127 opt.num_levels = 1;
128 opt.create_if_missing = true;
129 opt.max_open_files = 256;
130 opt.max_background_compactions = 10;
131 opt.arena_block_size = 8388608;
132 ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
133 {ColumnFamilyOptions(opt)}, kOptionsFileName,
134 Env::Default()));
135
136 // override the following options as db_bench will not take these
137 // options from the options file
138 opt.wal_dir = wal_path_;
139
140 RunDbBench(kOptionsFileName);
141
142 VerifyOptions(opt);
143 }
144
TEST_F(DBBenchTest,OptionsFileMultiLevelUniversal)145 TEST_F(DBBenchTest, OptionsFileMultiLevelUniversal) {
146 const std::string kOptionsFileName = test_path_ + "/OPTIONS_test";
147
148 Options opt;
149 opt.compaction_style = kCompactionStyleUniversal;
150 opt.num_levels = 12;
151 opt.create_if_missing = true;
152 opt.max_open_files = 256;
153 opt.max_background_compactions = 10;
154 opt.arena_block_size = 8388608;
155 ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
156 {ColumnFamilyOptions(opt)}, kOptionsFileName,
157 Env::Default()));
158
159 // override the following options as db_bench will not take these
160 // options from the options file
161 opt.wal_dir = wal_path_;
162
163 RunDbBench(kOptionsFileName);
164
165 VerifyOptions(opt);
166 }
167
168 const std::string options_file_content = R"OPTIONS_FILE(
169 [Version]
170 rocksdb_version=4.3.1
171 options_file_version=1.1
172
173 [DBOptions]
174 wal_bytes_per_sync=1048576
175 delete_obsolete_files_period_micros=0
176 WAL_ttl_seconds=0
177 WAL_size_limit_MB=0
178 db_write_buffer_size=0
179 max_subcompactions=1
180 table_cache_numshardbits=4
181 max_open_files=-1
182 max_file_opening_threads=10
183 max_background_compactions=5
184 use_fsync=false
185 use_adaptive_mutex=false
186 max_total_wal_size=18446744073709551615
187 compaction_readahead_size=0
188 new_table_reader_for_compaction_inputs=false
189 keep_log_file_num=10
190 skip_stats_update_on_db_open=false
191 max_manifest_file_size=18446744073709551615
192 db_log_dir=
193 skip_log_error_on_recovery=false
194 writable_file_max_buffer_size=1048576
195 paranoid_checks=true
196 is_fd_close_on_exec=true
197 bytes_per_sync=1048576
198 enable_thread_tracking=true
199 recycle_log_file_num=0
200 create_missing_column_families=false
201 log_file_time_to_roll=0
202 max_background_flushes=1
203 create_if_missing=true
204 error_if_exists=false
205 delayed_write_rate=1048576
206 manifest_preallocation_size=4194304
207 allow_mmap_reads=false
208 allow_mmap_writes=false
209 use_direct_reads=false
210 use_direct_io_for_flush_and_compaction=false
211 stats_dump_period_sec=600
212 allow_fallocate=true
213 max_log_file_size=83886080
214 random_access_max_buffer_size=1048576
215 advise_random_on_open=true
216
217
218 [CFOptions "default"]
219 compaction_filter_factory=nullptr
220 table_factory=BlockBasedTable
221 prefix_extractor=nullptr
222 comparator=leveldb.BytewiseComparator
223 compression_per_level=
224 max_bytes_for_level_base=104857600
225 bloom_locality=0
226 target_file_size_base=10485760
227 memtable_huge_page_size=0
228 max_successive_merges=1000
229 max_sequential_skip_in_iterations=8
230 arena_block_size=52428800
231 target_file_size_multiplier=1
232 source_compaction_factor=1
233 min_write_buffer_number_to_merge=1
234 max_write_buffer_number=2
235 write_buffer_size=419430400
236 max_grandparent_overlap_factor=10
237 max_bytes_for_level_multiplier=10
238 memtable_factory=SkipListFactory
239 compression=kSnappyCompression
240 min_partial_merge_operands=2
241 level0_stop_writes_trigger=100
242 num_levels=1
243 level0_slowdown_writes_trigger=50
244 level0_file_num_compaction_trigger=10
245 expanded_compaction_factor=25
246 soft_rate_limit=0.000000
247 max_write_buffer_number_to_maintain=0
248 max_write_buffer_size_to_maintain=0
249 verify_checksums_in_compaction=true
250 merge_operator=nullptr
251 memtable_prefix_bloom_bits=0
252 memtable_whole_key_filtering=true
253 paranoid_file_checks=false
254 inplace_update_num_locks=10000
255 optimize_filters_for_hits=false
256 level_compaction_dynamic_level_bytes=false
257 inplace_update_support=false
258 compaction_style=kCompactionStyleUniversal
259 memtable_prefix_bloom_probes=6
260 purge_redundant_kvs_while_flush=true
261 filter_deletes=false
262 hard_pending_compaction_bytes_limit=0
263 disable_auto_compactions=false
264 compaction_measure_io_stats=false
265
266 [TableOptions/BlockBasedTable "default"]
267 format_version=0
268 skip_table_builder_flush=false
269 cache_index_and_filter_blocks=false
270 flush_block_policy_factory=FlushBlockBySizePolicyFactory
271 hash_index_allow_collision=true
272 index_type=kBinarySearch
273 whole_key_filtering=true
274 checksum=kCRC32c
275 no_block_cache=false
276 block_size=32768
277 block_size_deviation=10
278 block_restart_interval=16
279 filter_policy=rocksdb.BuiltinBloomFilter
280 )OPTIONS_FILE";
281
TEST_F(DBBenchTest,OptionsFileFromFile)282 TEST_F(DBBenchTest, OptionsFileFromFile) {
283 const std::string kOptionsFileName = test_path_ + "/OPTIONS_flash";
284 std::unique_ptr<WritableFile> writable;
285 ASSERT_OK(Env::Default()->NewWritableFile(kOptionsFileName, &writable,
286 EnvOptions()));
287 ASSERT_OK(writable->Append(options_file_content));
288 ASSERT_OK(writable->Close());
289
290 DBOptions db_opt;
291 std::vector<ColumnFamilyDescriptor> cf_descs;
292 ASSERT_OK(LoadOptionsFromFile(kOptionsFileName, Env::Default(), &db_opt,
293 &cf_descs));
294 Options opt(db_opt, cf_descs[0].options);
295
296 opt.create_if_missing = true;
297
298 // override the following options as db_bench will not take these
299 // options from the options file
300 opt.wal_dir = wal_path_;
301
302 RunDbBench(kOptionsFileName);
303
304 VerifyOptions(opt);
305 }
306
307 } // namespace ROCKSDB_NAMESPACE
308
main(int argc,char ** argv)309 int main(int argc, char** argv) {
310 ::testing::InitGoogleTest(&argc, argv);
311 google::ParseCommandLineFlags(&argc, &argv, true);
312 return RUN_ALL_TESTS();
313 }
314
315 #else
316
main(int argc,char ** argv)317 int main(int argc, char** argv) {
318 printf("Skip db_bench_tool_test as the required library GFLAG is missing.");
319 }
320 #endif // #ifdef GFLAGS
321