1 // Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2 //  This source code is licensed under both the GPLv2 (found in the
3 //  COPYING file in the root directory) and Apache 2.0 License
4 //  (found in the LICENSE.Apache file in the root directory).
5 
6 #ifndef ROCKSDB_LITE
7 
8 #include "options/options_parser.h"
9 
10 #include <cmath>
11 #include <map>
12 #include <string>
13 #include <utility>
14 #include <vector>
15 
16 #include "file/read_write_util.h"
17 #include "file/writable_file_writer.h"
18 #include "options/options_helper.h"
19 #include "rocksdb/convenience.h"
20 #include "rocksdb/db.h"
21 #include "test_util/sync_point.h"
22 #include "util/cast_util.h"
23 #include "util/string_util.h"
24 
25 #include "port/port.h"
26 
27 namespace ROCKSDB_NAMESPACE {
28 
29 static const std::string option_file_header =
30     "# This is a RocksDB option file.\n"
31     "#\n"
32     "# For detailed file format spec, please refer to the example file\n"
33     "# in examples/rocksdb_option_file_example.ini\n"
34     "#\n"
35     "\n";
36 
PersistRocksDBOptions(const DBOptions & db_opt,const std::vector<std::string> & cf_names,const std::vector<ColumnFamilyOptions> & cf_opts,const std::string & file_name,FileSystem * fs)37 Status PersistRocksDBOptions(const DBOptions& db_opt,
38                              const std::vector<std::string>& cf_names,
39                              const std::vector<ColumnFamilyOptions>& cf_opts,
40                              const std::string& file_name, FileSystem* fs) {
41   TEST_SYNC_POINT("PersistRocksDBOptions:start");
42   if (cf_names.size() != cf_opts.size()) {
43     return Status::InvalidArgument(
44         "cf_names.size() and cf_opts.size() must be the same");
45   }
46   std::unique_ptr<FSWritableFile> wf;
47 
48   Status s =
49       fs->NewWritableFile(file_name, FileOptions(), &wf, nullptr);
50   if (!s.ok()) {
51     return s;
52   }
53   std::unique_ptr<WritableFileWriter> writable;
54   writable.reset(new WritableFileWriter(std::move(wf), file_name, EnvOptions(),
55                                         nullptr /* statistics */));
56 
57   std::string options_file_content;
58 
59   writable->Append(option_file_header + "[" +
60                    opt_section_titles[kOptionSectionVersion] +
61                    "]\n"
62                    "  rocksdb_version=" +
63                    ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR) +
64                    "." + ToString(ROCKSDB_PATCH) + "\n");
65   writable->Append("  options_file_version=" +
66                    ToString(ROCKSDB_OPTION_FILE_MAJOR) + "." +
67                    ToString(ROCKSDB_OPTION_FILE_MINOR) + "\n");
68   writable->Append("\n[" + opt_section_titles[kOptionSectionDBOptions] +
69                    "]\n  ");
70 
71   s = GetStringFromDBOptions(&options_file_content, db_opt, "\n  ");
72   if (!s.ok()) {
73     writable->Close();
74     return s;
75   }
76   writable->Append(options_file_content + "\n");
77 
78   for (size_t i = 0; i < cf_opts.size(); ++i) {
79     // CFOptions section
80     writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] +
81                      " \"" + EscapeOptionString(cf_names[i]) + "\"]\n  ");
82     s = GetStringFromColumnFamilyOptions(&options_file_content, cf_opts[i],
83                                          "\n  ");
84     if (!s.ok()) {
85       writable->Close();
86       return s;
87     }
88     writable->Append(options_file_content + "\n");
89     // TableOptions section
90     auto* tf = cf_opts[i].table_factory.get();
91     if (tf != nullptr) {
92       writable->Append("[" + opt_section_titles[kOptionSectionTableOptions] +
93                        tf->Name() + " \"" + EscapeOptionString(cf_names[i]) +
94                        "\"]\n  ");
95       options_file_content.clear();
96       s = tf->GetOptionString(&options_file_content, "\n  ");
97       if (!s.ok()) {
98         return s;
99       }
100       writable->Append(options_file_content + "\n");
101     }
102   }
103   writable->Sync(true /* use_fsync */);
104   writable->Close();
105 
106   return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
107       db_opt, cf_names, cf_opts, file_name, fs);
108 }
109 
RocksDBOptionsParser()110 RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); }
111 
Reset()112 void RocksDBOptionsParser::Reset() {
113   db_opt_ = DBOptions();
114   db_opt_map_.clear();
115   cf_names_.clear();
116   cf_opts_.clear();
117   cf_opt_maps_.clear();
118   has_version_section_ = false;
119   has_db_options_ = false;
120   has_default_cf_options_ = false;
121   for (int i = 0; i < 3; ++i) {
122     db_version[i] = 0;
123     opt_file_version[i] = 0;
124   }
125 }
126 
IsSection(const std::string & line)127 bool RocksDBOptionsParser::IsSection(const std::string& line) {
128   if (line.size() < 2) {
129     return false;
130   }
131   if (line[0] != '[' || line[line.size() - 1] != ']') {
132     return false;
133   }
134   return true;
135 }
136 
ParseSection(OptionSection * section,std::string * title,std::string * argument,const std::string & line,const int line_num)137 Status RocksDBOptionsParser::ParseSection(OptionSection* section,
138                                           std::string* title,
139                                           std::string* argument,
140                                           const std::string& line,
141                                           const int line_num) {
142   *section = kOptionSectionUnknown;
143   // A section is of the form [<SectionName> "<SectionArg>"], where
144   // "<SectionArg>" is optional.
145   size_t arg_start_pos = line.find("\"");
146   size_t arg_end_pos = line.rfind("\"");
147   // The following if-then check tries to identify whether the input
148   // section has the optional section argument.
149   if (arg_start_pos != std::string::npos && arg_start_pos != arg_end_pos) {
150     *title = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true);
151     *argument = UnescapeOptionString(
152         line.substr(arg_start_pos + 1, arg_end_pos - arg_start_pos - 1));
153   } else {
154     *title = TrimAndRemoveComment(line.substr(1, line.size() - 2), true);
155     *argument = "";
156   }
157   for (int i = 0; i < kOptionSectionUnknown; ++i) {
158     if (title->find(opt_section_titles[i]) == 0) {
159       if (i == kOptionSectionVersion || i == kOptionSectionDBOptions ||
160           i == kOptionSectionCFOptions) {
161         if (title->size() == opt_section_titles[i].size()) {
162           // if true, then it indicats equal
163           *section = static_cast<OptionSection>(i);
164           return CheckSection(*section, *argument, line_num);
165         }
166       } else if (i == kOptionSectionTableOptions) {
167         // This type of sections has a sufffix at the end of the
168         // section title
169         if (title->size() > opt_section_titles[i].size()) {
170           *section = static_cast<OptionSection>(i);
171           return CheckSection(*section, *argument, line_num);
172         }
173       }
174     }
175   }
176   return Status::InvalidArgument(std::string("Unknown section ") + line);
177 }
178 
InvalidArgument(const int line_num,const std::string & message)179 Status RocksDBOptionsParser::InvalidArgument(const int line_num,
180                                              const std::string& message) {
181   return Status::InvalidArgument(
182       "[RocksDBOptionsParser Error] ",
183       message + " (at line " + ToString(line_num) + ")");
184 }
185 
ParseStatement(std::string * name,std::string * value,const std::string & line,const int line_num)186 Status RocksDBOptionsParser::ParseStatement(std::string* name,
187                                             std::string* value,
188                                             const std::string& line,
189                                             const int line_num) {
190   size_t eq_pos = line.find("=");
191   if (eq_pos == std::string::npos) {
192     return InvalidArgument(line_num, "A valid statement must have a '='.");
193   }
194 
195   *name = TrimAndRemoveComment(line.substr(0, eq_pos), true);
196   *value =
197       TrimAndRemoveComment(line.substr(eq_pos + 1, line.size() - eq_pos - 1));
198   if (name->empty()) {
199     return InvalidArgument(line_num,
200                            "A valid statement must have a variable name.");
201   }
202   return Status::OK();
203 }
204 
Parse(const std::string & file_name,FileSystem * fs,bool ignore_unknown_options,size_t file_readahead_size)205 Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs,
206                                    bool ignore_unknown_options,
207                                    size_t file_readahead_size) {
208   Reset();
209 
210   std::unique_ptr<FSSequentialFile> seq_file;
211   Status s = fs->NewSequentialFile(file_name, FileOptions(), &seq_file,
212                                    nullptr);
213   if (!s.ok()) {
214     return s;
215   }
216 
217   SequentialFileReader sf_reader(std::move(seq_file), file_name,
218                                  file_readahead_size);
219 
220   OptionSection section = kOptionSectionUnknown;
221   std::string title;
222   std::string argument;
223   std::unordered_map<std::string, std::string> opt_map;
224   std::istringstream iss;
225   std::string line;
226   bool has_data = true;
227   // we only support single-lined statement.
228   for (int line_num = 1; ReadOneLine(&iss, &sf_reader, &line, &has_data, &s);
229        ++line_num) {
230     if (!s.ok()) {
231       return s;
232     }
233     line = TrimAndRemoveComment(line);
234     if (line.empty()) {
235       continue;
236     }
237     if (IsSection(line)) {
238       s = EndSection(section, title, argument, opt_map, ignore_unknown_options);
239       opt_map.clear();
240       if (!s.ok()) {
241         return s;
242       }
243 
244       // If the option file is not generated by a higher minor version,
245       // there shouldn't be any unknown option.
246       if (ignore_unknown_options && section == kOptionSectionVersion) {
247         if (db_version[0] < ROCKSDB_MAJOR || (db_version[0] == ROCKSDB_MAJOR &&
248                                               db_version[1] <= ROCKSDB_MINOR)) {
249           ignore_unknown_options = false;
250         }
251       }
252 
253       s = ParseSection(&section, &title, &argument, line, line_num);
254       if (!s.ok()) {
255         return s;
256       }
257     } else {
258       std::string name;
259       std::string value;
260       s = ParseStatement(&name, &value, line, line_num);
261       if (!s.ok()) {
262         return s;
263       }
264       opt_map.insert({name, value});
265     }
266   }
267 
268   s = EndSection(section, title, argument, opt_map, ignore_unknown_options);
269   opt_map.clear();
270   if (!s.ok()) {
271     return s;
272   }
273   return ValidityCheck();
274 }
275 
CheckSection(const OptionSection section,const std::string & section_arg,const int line_num)276 Status RocksDBOptionsParser::CheckSection(const OptionSection section,
277                                           const std::string& section_arg,
278                                           const int line_num) {
279   if (section == kOptionSectionDBOptions) {
280     if (has_db_options_) {
281       return InvalidArgument(
282           line_num,
283           "More than one DBOption section found in the option config file");
284     }
285     has_db_options_ = true;
286   } else if (section == kOptionSectionCFOptions) {
287     bool is_default_cf = (section_arg == kDefaultColumnFamilyName);
288     if (cf_opts_.size() == 0 && !is_default_cf) {
289       return InvalidArgument(
290           line_num,
291           "Default column family must be the first CFOptions section "
292           "in the option config file");
293     } else if (cf_opts_.size() != 0 && is_default_cf) {
294       return InvalidArgument(
295           line_num,
296           "Default column family must be the first CFOptions section "
297           "in the optio/n config file");
298     } else if (GetCFOptions(section_arg) != nullptr) {
299       return InvalidArgument(
300           line_num,
301           "Two identical column families found in option config file");
302     }
303     has_default_cf_options_ |= is_default_cf;
304   } else if (section == kOptionSectionTableOptions) {
305     if (GetCFOptions(section_arg) == nullptr) {
306       return InvalidArgument(
307           line_num, std::string(
308                         "Does not find a matched column family name in "
309                         "TableOptions section.  Column Family Name:") +
310                         section_arg);
311     }
312   } else if (section == kOptionSectionVersion) {
313     if (has_version_section_) {
314       return InvalidArgument(
315           line_num,
316           "More than one Version section found in the option config file.");
317     }
318     has_version_section_ = true;
319   }
320   return Status::OK();
321 }
322 
ParseVersionNumber(const std::string & ver_name,const std::string & ver_string,const int max_count,int * version)323 Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name,
324                                                 const std::string& ver_string,
325                                                 const int max_count,
326                                                 int* version) {
327   int version_index = 0;
328   int current_number = 0;
329   int current_digit_count = 0;
330   bool has_dot = false;
331   for (int i = 0; i < max_count; ++i) {
332     version[i] = 0;
333   }
334   constexpr int kBufferSize = 200;
335   char buffer[kBufferSize];
336   for (size_t i = 0; i < ver_string.size(); ++i) {
337     if (ver_string[i] == '.') {
338       if (version_index >= max_count - 1) {
339         snprintf(buffer, sizeof(buffer) - 1,
340                  "A valid %s can only contains at most %d dots.",
341                  ver_name.c_str(), max_count - 1);
342         return Status::InvalidArgument(buffer);
343       }
344       if (current_digit_count == 0) {
345         snprintf(buffer, sizeof(buffer) - 1,
346                  "A valid %s must have at least one digit before each dot.",
347                  ver_name.c_str());
348         return Status::InvalidArgument(buffer);
349       }
350       version[version_index++] = current_number;
351       current_number = 0;
352       current_digit_count = 0;
353       has_dot = true;
354     } else if (isdigit(ver_string[i])) {
355       current_number = current_number * 10 + (ver_string[i] - '0');
356       current_digit_count++;
357     } else {
358       snprintf(buffer, sizeof(buffer) - 1,
359                "A valid %s can only contains dots and numbers.",
360                ver_name.c_str());
361       return Status::InvalidArgument(buffer);
362     }
363   }
364   version[version_index] = current_number;
365   if (has_dot && current_digit_count == 0) {
366     snprintf(buffer, sizeof(buffer) - 1,
367              "A valid %s must have at least one digit after each dot.",
368              ver_name.c_str());
369     return Status::InvalidArgument(buffer);
370   }
371   return Status::OK();
372 }
373 
EndSection(const OptionSection section,const std::string & section_title,const std::string & section_arg,const std::unordered_map<std::string,std::string> & opt_map,bool ignore_unknown_options)374 Status RocksDBOptionsParser::EndSection(
375     const OptionSection section, const std::string& section_title,
376     const std::string& section_arg,
377     const std::unordered_map<std::string, std::string>& opt_map,
378     bool ignore_unknown_options) {
379   Status s;
380   if (section == kOptionSectionDBOptions) {
381     s = GetDBOptionsFromMap(DBOptions(), opt_map, &db_opt_, true,
382                             ignore_unknown_options);
383     if (!s.ok()) {
384       return s;
385     }
386     db_opt_map_ = opt_map;
387   } else if (section == kOptionSectionCFOptions) {
388     // This condition should be ensured earlier in ParseSection
389     // so we make an assertion here.
390     assert(GetCFOptions(section_arg) == nullptr);
391     cf_names_.emplace_back(section_arg);
392     cf_opts_.emplace_back();
393     s = GetColumnFamilyOptionsFromMap(ColumnFamilyOptions(), opt_map,
394                                       &cf_opts_.back(), true,
395                                       ignore_unknown_options);
396     if (!s.ok()) {
397       return s;
398     }
399     // keep the parsed string.
400     cf_opt_maps_.emplace_back(opt_map);
401   } else if (section == kOptionSectionTableOptions) {
402     assert(GetCFOptions(section_arg) != nullptr);
403     auto* cf_opt = GetCFOptionsImpl(section_arg);
404     if (cf_opt == nullptr) {
405       return Status::InvalidArgument(
406           "The specified column family must be defined before the "
407           "TableOptions section:",
408           section_arg);
409     }
410     // Ignore error as table factory deserialization is optional
411     s = GetTableFactoryFromMap(
412         section_title.substr(
413             opt_section_titles[kOptionSectionTableOptions].size()),
414         opt_map, &(cf_opt->table_factory), ignore_unknown_options);
415     if (!s.ok()) {
416       return s;
417     }
418   } else if (section == kOptionSectionVersion) {
419     for (const auto& pair : opt_map) {
420       if (pair.first == "rocksdb_version") {
421         s = ParseVersionNumber(pair.first, pair.second, 3, db_version);
422         if (!s.ok()) {
423           return s;
424         }
425       } else if (pair.first == "options_file_version") {
426         s = ParseVersionNumber(pair.first, pair.second, 2, opt_file_version);
427         if (!s.ok()) {
428           return s;
429         }
430         if (opt_file_version[0] < 1) {
431           return Status::InvalidArgument(
432               "A valid options_file_version must be at least 1.");
433         }
434       }
435     }
436   }
437   return Status::OK();
438 }
439 
ValidityCheck()440 Status RocksDBOptionsParser::ValidityCheck() {
441   if (!has_db_options_) {
442     return Status::Corruption(
443         "A RocksDB Option file must have a single DBOptions section");
444   }
445   if (!has_default_cf_options_) {
446     return Status::Corruption(
447         "A RocksDB Option file must have a single CFOptions:default section");
448   }
449 
450   return Status::OK();
451 }
452 
TrimAndRemoveComment(const std::string & line,bool trim_only)453 std::string RocksDBOptionsParser::TrimAndRemoveComment(const std::string& line,
454                                                        bool trim_only) {
455   size_t start = 0;
456   size_t end = line.size();
457 
458   // we only support "#" style comment
459   if (!trim_only) {
460     size_t search_pos = 0;
461     while (search_pos < line.size()) {
462       size_t comment_pos = line.find('#', search_pos);
463       if (comment_pos == std::string::npos) {
464         break;
465       }
466       if (comment_pos == 0 || line[comment_pos - 1] != '\\') {
467         end = comment_pos;
468         break;
469       }
470       search_pos = comment_pos + 1;
471     }
472   }
473 
474   while (start < end && isspace(line[start]) != 0) {
475     ++start;
476   }
477 
478   // start < end implies end > 0.
479   while (start < end && isspace(line[end - 1]) != 0) {
480     --end;
481   }
482 
483   if (start < end) {
484     return line.substr(start, end - start);
485   }
486 
487   return "";
488 }
489 
490 namespace {
AreEqualDoubles(const double a,const double b)491 bool AreEqualDoubles(const double a, const double b) {
492   return (fabs(a - b) < 0.00001);
493 }
494 }  // namespace
495 
AreEqualOptions(const char * opt1,const char * opt2,const OptionTypeInfo & type_info,const std::string & opt_name,const std::unordered_map<std::string,std::string> * opt_map)496 bool AreEqualOptions(
497     const char* opt1, const char* opt2, const OptionTypeInfo& type_info,
498     const std::string& opt_name,
499     const std::unordered_map<std::string, std::string>* opt_map) {
500   const char* offset1 = opt1 + type_info.offset;
501   const char* offset2 = opt2 + type_info.offset;
502 
503   switch (type_info.type) {
504     case OptionType::kBoolean:
505       return (*reinterpret_cast<const bool*>(offset1) ==
506               *reinterpret_cast<const bool*>(offset2));
507     case OptionType::kInt:
508       return (*reinterpret_cast<const int*>(offset1) ==
509               *reinterpret_cast<const int*>(offset2));
510     case OptionType::kInt32T:
511       return (*reinterpret_cast<const int32_t*>(offset1) ==
512               *reinterpret_cast<const int32_t*>(offset2));
513     case OptionType::kInt64T:
514       {
515         int64_t v1, v2;
516         GetUnaligned(reinterpret_cast<const int64_t*>(offset1), &v1);
517         GetUnaligned(reinterpret_cast<const int64_t*>(offset2), &v2);
518         return (v1 == v2);
519       }
520     case OptionType::kVectorInt:
521       return (*reinterpret_cast<const std::vector<int>*>(offset1) ==
522               *reinterpret_cast<const std::vector<int>*>(offset2));
523     case OptionType::kUInt:
524       return (*reinterpret_cast<const unsigned int*>(offset1) ==
525               *reinterpret_cast<const unsigned int*>(offset2));
526     case OptionType::kUInt32T:
527       return (*reinterpret_cast<const uint32_t*>(offset1) ==
528               *reinterpret_cast<const uint32_t*>(offset2));
529     case OptionType::kUInt64T:
530       {
531         uint64_t v1, v2;
532         GetUnaligned(reinterpret_cast<const uint64_t*>(offset1), &v1);
533         GetUnaligned(reinterpret_cast<const uint64_t*>(offset2), &v2);
534         return (v1 == v2);
535       }
536     case OptionType::kSizeT:
537       {
538         size_t v1, v2;
539         GetUnaligned(reinterpret_cast<const size_t*>(offset1), &v1);
540         GetUnaligned(reinterpret_cast<const size_t*>(offset2), &v2);
541         return (v1 == v2);
542       }
543     case OptionType::kString:
544       return (*reinterpret_cast<const std::string*>(offset1) ==
545               *reinterpret_cast<const std::string*>(offset2));
546     case OptionType::kDouble:
547       return AreEqualDoubles(*reinterpret_cast<const double*>(offset1),
548                              *reinterpret_cast<const double*>(offset2));
549     case OptionType::kCompactionStyle:
550       return (*reinterpret_cast<const CompactionStyle*>(offset1) ==
551               *reinterpret_cast<const CompactionStyle*>(offset2));
552     case OptionType::kCompactionPri:
553       return (*reinterpret_cast<const CompactionPri*>(offset1) ==
554               *reinterpret_cast<const CompactionPri*>(offset2));
555     case OptionType::kCompressionType:
556       return (*reinterpret_cast<const CompressionType*>(offset1) ==
557               *reinterpret_cast<const CompressionType*>(offset2));
558     case OptionType::kVectorCompressionType: {
559       const auto* vec1 =
560           reinterpret_cast<const std::vector<CompressionType>*>(offset1);
561       const auto* vec2 =
562           reinterpret_cast<const std::vector<CompressionType>*>(offset2);
563       return (*vec1 == *vec2);
564     }
565     case OptionType::kChecksumType:
566       return (*reinterpret_cast<const ChecksumType*>(offset1) ==
567               *reinterpret_cast<const ChecksumType*>(offset2));
568     case OptionType::kBlockBasedTableIndexType:
569       return (
570           *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(
571               offset1) ==
572           *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(offset2));
573     case OptionType::kBlockBasedTableDataBlockIndexType:
574       return (
575           *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>(
576               offset1) ==
577           *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>(
578               offset2));
579     case OptionType::kBlockBasedTableIndexShorteningMode:
580       return (
581           *reinterpret_cast<const BlockBasedTableOptions::IndexShorteningMode*>(
582               offset1) ==
583           *reinterpret_cast<const BlockBasedTableOptions::IndexShorteningMode*>(
584               offset2));
585     case OptionType::kWALRecoveryMode:
586       return (*reinterpret_cast<const WALRecoveryMode*>(offset1) ==
587               *reinterpret_cast<const WALRecoveryMode*>(offset2));
588     case OptionType::kAccessHint:
589       return (*reinterpret_cast<const DBOptions::AccessHint*>(offset1) ==
590               *reinterpret_cast<const DBOptions::AccessHint*>(offset2));
591     case OptionType::kInfoLogLevel:
592       return (*reinterpret_cast<const InfoLogLevel*>(offset1) ==
593               *reinterpret_cast<const InfoLogLevel*>(offset2));
594     case OptionType::kCompactionOptionsFIFO: {
595       CompactionOptionsFIFO lhs =
596           *reinterpret_cast<const CompactionOptionsFIFO*>(offset1);
597       CompactionOptionsFIFO rhs =
598           *reinterpret_cast<const CompactionOptionsFIFO*>(offset2);
599       if (lhs.max_table_files_size == rhs.max_table_files_size &&
600           lhs.allow_compaction == rhs.allow_compaction) {
601         return true;
602       }
603       return false;
604     }
605     case OptionType::kCompactionOptionsUniversal: {
606       CompactionOptionsUniversal lhs =
607           *reinterpret_cast<const CompactionOptionsUniversal*>(offset1);
608       CompactionOptionsUniversal rhs =
609           *reinterpret_cast<const CompactionOptionsUniversal*>(offset2);
610       if (lhs.size_ratio == rhs.size_ratio &&
611           lhs.min_merge_width == rhs.min_merge_width &&
612           lhs.max_merge_width == rhs.max_merge_width &&
613           lhs.max_size_amplification_percent ==
614               rhs.max_size_amplification_percent &&
615           lhs.compression_size_percent == rhs.compression_size_percent &&
616           lhs.stop_style == rhs.stop_style &&
617           lhs.allow_trivial_move == rhs.allow_trivial_move) {
618         return true;
619       }
620       return false;
621     }
622     default:
623       if (type_info.verification == OptionVerificationType::kByName ||
624           type_info.verification ==
625               OptionVerificationType::kByNameAllowFromNull ||
626           type_info.verification == OptionVerificationType::kByNameAllowNull) {
627         std::string value1;
628         bool result =
629             SerializeSingleOptionHelper(offset1, type_info.type, &value1);
630         if (result == false) {
631           return false;
632         }
633         if (opt_map == nullptr) {
634           return true;
635         }
636         auto iter = opt_map->find(opt_name);
637         if (iter == opt_map->end()) {
638           return true;
639         } else {
640           if (type_info.verification ==
641               OptionVerificationType::kByNameAllowNull) {
642             if (iter->second == kNullptrString || value1 == kNullptrString) {
643               return true;
644             }
645           } else if (type_info.verification ==
646                      OptionVerificationType::kByNameAllowFromNull) {
647             if (iter->second == kNullptrString) {
648               return true;
649             }
650           }
651           return (value1 == iter->second);
652         }
653       }
654       return false;
655   }
656 }
657 
VerifyRocksDBOptionsFromFile(const DBOptions & db_opt,const std::vector<std::string> & cf_names,const std::vector<ColumnFamilyOptions> & cf_opts,const std::string & file_name,FileSystem * fs,OptionsSanityCheckLevel sanity_check_level,bool ignore_unknown_options)658 Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
659     const DBOptions& db_opt, const std::vector<std::string>& cf_names,
660     const std::vector<ColumnFamilyOptions>& cf_opts,
661     const std::string& file_name, FileSystem* fs,
662     OptionsSanityCheckLevel sanity_check_level, bool ignore_unknown_options) {
663   // We infer option file readhead size from log readahead size.
664   // If it is not given, use 512KB.
665   size_t file_readahead_size = db_opt.log_readahead_size;
666   if (file_readahead_size == 0) {
667     const size_t kDefaultOptionFileReadAheadSize = 512 * 1024;
668     file_readahead_size = kDefaultOptionFileReadAheadSize;
669   }
670 
671   RocksDBOptionsParser parser;
672   Status s =
673       parser.Parse(file_name, fs, ignore_unknown_options, file_readahead_size);
674   if (!s.ok()) {
675     return s;
676   }
677 
678   // Verify DBOptions
679   s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map(),
680                       sanity_check_level);
681   if (!s.ok()) {
682     return s;
683   }
684 
685   // Verify ColumnFamily Name
686   if (cf_names.size() != parser.cf_names()->size()) {
687     if (sanity_check_level >= kSanityLevelLooselyCompatible) {
688       return Status::InvalidArgument(
689           "[RocksDBOptionParser Error] The persisted options does not have "
690           "the same number of column family names as the db instance.");
691     } else if (cf_opts.size() > parser.cf_opts()->size()) {
692       return Status::InvalidArgument(
693           "[RocksDBOptionsParser Error]",
694           "The persisted options file has less number of column family "
695           "names than that of the specified one.");
696     }
697   }
698   for (size_t i = 0; i < cf_names.size(); ++i) {
699     if (cf_names[i] != parser.cf_names()->at(i)) {
700       return Status::InvalidArgument(
701           "[RocksDBOptionParser Error] The persisted options and the db"
702           "instance does not have the same name for column family ",
703           ToString(i));
704     }
705   }
706 
707   // Verify Column Family Options
708   if (cf_opts.size() != parser.cf_opts()->size()) {
709     if (sanity_check_level >= kSanityLevelLooselyCompatible) {
710       return Status::InvalidArgument(
711           "[RocksDBOptionsParser Error]",
712           "The persisted options does not have the same number of "
713           "column families as the db instance.");
714     } else if (cf_opts.size() > parser.cf_opts()->size()) {
715       return Status::InvalidArgument(
716           "[RocksDBOptionsParser Error]",
717           "The persisted options file has less number of column families "
718           "than that of the specified number.");
719     }
720   }
721   for (size_t i = 0; i < cf_opts.size(); ++i) {
722     s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i),
723                         &(parser.cf_opt_maps()->at(i)), sanity_check_level);
724     if (!s.ok()) {
725       return s;
726     }
727     s = VerifyTableFactory(cf_opts[i].table_factory.get(),
728                            parser.cf_opts()->at(i).table_factory.get(),
729                            sanity_check_level);
730     if (!s.ok()) {
731       return s;
732     }
733   }
734 
735   return Status::OK();
736 }
737 
VerifyDBOptions(const DBOptions & base_opt,const DBOptions & persisted_opt,const std::unordered_map<std::string,std::string> *,OptionsSanityCheckLevel sanity_check_level)738 Status RocksDBOptionsParser::VerifyDBOptions(
739     const DBOptions& base_opt, const DBOptions& persisted_opt,
740     const std::unordered_map<std::string, std::string>* /*opt_map*/,
741     OptionsSanityCheckLevel sanity_check_level) {
742   for (const auto& pair : db_options_type_info) {
743     if (pair.second.verification == OptionVerificationType::kDeprecated) {
744       // We skip checking deprecated variables as they might
745       // contain random values since they might not be initialized
746       continue;
747     }
748     if (DBOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
749       if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
750                            reinterpret_cast<const char*>(&persisted_opt),
751                            pair.second, pair.first, nullptr)) {
752         constexpr size_t kBufferSize = 2048;
753         char buffer[kBufferSize];
754         std::string base_value;
755         std::string persisted_value;
756         SerializeSingleOptionHelper(
757             reinterpret_cast<const char*>(&base_opt) + pair.second.offset,
758             pair.second.type, &base_value);
759         SerializeSingleOptionHelper(
760             reinterpret_cast<const char*>(&persisted_opt) + pair.second.offset,
761             pair.second.type, &persisted_value);
762         snprintf(buffer, sizeof(buffer),
763                  "[RocksDBOptionsParser]: "
764                  "failed the verification on DBOptions::%s --- "
765                  "The specified one is %s while the persisted one is %s.\n",
766                  pair.first.c_str(), base_value.c_str(),
767                  persisted_value.c_str());
768         return Status::InvalidArgument(Slice(buffer, strlen(buffer)));
769       }
770     }
771   }
772   return Status::OK();
773 }
774 
VerifyCFOptions(const ColumnFamilyOptions & base_opt,const ColumnFamilyOptions & persisted_opt,const std::unordered_map<std::string,std::string> * persisted_opt_map,OptionsSanityCheckLevel sanity_check_level)775 Status RocksDBOptionsParser::VerifyCFOptions(
776     const ColumnFamilyOptions& base_opt,
777     const ColumnFamilyOptions& persisted_opt,
778     const std::unordered_map<std::string, std::string>* persisted_opt_map,
779     OptionsSanityCheckLevel sanity_check_level) {
780   for (const auto& pair : cf_options_type_info) {
781     if (pair.second.verification == OptionVerificationType::kDeprecated) {
782       // We skip checking deprecated variables as they might
783       // contain random values since they might not be initialized
784       continue;
785     }
786     if (CFOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
787       if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
788                            reinterpret_cast<const char*>(&persisted_opt),
789                            pair.second, pair.first, persisted_opt_map)) {
790         constexpr size_t kBufferSize = 2048;
791         char buffer[kBufferSize];
792         std::string base_value;
793         std::string persisted_value;
794         SerializeSingleOptionHelper(
795             reinterpret_cast<const char*>(&base_opt) + pair.second.offset,
796             pair.second.type, &base_value);
797         SerializeSingleOptionHelper(
798             reinterpret_cast<const char*>(&persisted_opt) + pair.second.offset,
799             pair.second.type, &persisted_value);
800         snprintf(buffer, sizeof(buffer),
801                  "[RocksDBOptionsParser]: "
802                  "failed the verification on ColumnFamilyOptions::%s --- "
803                  "The specified one is %s while the persisted one is %s.\n",
804                  pair.first.c_str(), base_value.c_str(),
805                  persisted_value.c_str());
806         return Status::InvalidArgument(Slice(buffer, sizeof(buffer)));
807       }
808     }
809   }
810   return Status::OK();
811 }
812 
VerifyTableFactory(const TableFactory * base_tf,const TableFactory * file_tf,OptionsSanityCheckLevel sanity_check_level)813 Status RocksDBOptionsParser::VerifyTableFactory(
814     const TableFactory* base_tf, const TableFactory* file_tf,
815     OptionsSanityCheckLevel sanity_check_level) {
816   if (base_tf && file_tf) {
817     if (sanity_check_level > kSanityLevelNone &&
818         std::string(base_tf->Name()) != std::string(file_tf->Name())) {
819       return Status::Corruption(
820           "[RocksDBOptionsParser]: "
821           "failed the verification on TableFactory->Name()");
822     }
823     if (base_tf->Name() == BlockBasedTableFactory::kName) {
824       return VerifyBlockBasedTableFactory(
825           static_cast_with_check<const BlockBasedTableFactory,
826                                  const TableFactory>(base_tf),
827           static_cast_with_check<const BlockBasedTableFactory,
828                                  const TableFactory>(file_tf),
829           sanity_check_level);
830     }
831     // TODO(yhchiang): add checks for other table factory types
832   } else {
833     // TODO(yhchiang): further support sanity check here
834   }
835   return Status::OK();
836 }
837 }  // namespace ROCKSDB_NAMESPACE
838 
839 #endif  // !ROCKSDB_LITE
840