1
2 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
3 // This source code is licensed under both the GPLv2 (found in the
4 // COPYING file in the root directory) and Apache 2.0 License
5 // (found in the LICENSE.Apache file in the root directory).
6 //
7 #ifndef ROCKSDB_LITE
8 #include "rocksdb/utilities/ldb_cmd.h"
9
10 #include <cinttypes>
11
12 #include "db/db_impl/db_impl.h"
13 #include "db/dbformat.h"
14 #include "db/log_reader.h"
15 #include "db/write_batch_internal.h"
16 #include "env/composite_env_wrapper.h"
17 #include "file/filename.h"
18 #include "port/port_dirent.h"
19 #include "rocksdb/cache.h"
20 #include "rocksdb/file_checksum.h"
21 #include "rocksdb/table_properties.h"
22 #include "rocksdb/utilities/backupable_db.h"
23 #include "rocksdb/utilities/checkpoint.h"
24 #include "rocksdb/utilities/debug.h"
25 #include "rocksdb/utilities/options_util.h"
26 #include "rocksdb/write_batch.h"
27 #include "rocksdb/write_buffer_manager.h"
28 #include "table/scoped_arena_iterator.h"
29 #include "tools/ldb_cmd_impl.h"
30 #include "tools/sst_dump_tool_imp.h"
31 #include "util/cast_util.h"
32 #include "util/coding.h"
33 #include "util/file_checksum_helper.h"
34 #include "util/stderr_logger.h"
35 #include "util/string_util.h"
36 #include "utilities/merge_operators.h"
37 #include "utilities/ttl/db_ttl_impl.h"
38
39 #include <cstdlib>
40 #include <ctime>
41 #include <fstream>
42 #include <functional>
43 #include <iostream>
44 #include <limits>
45 #include <sstream>
46 #include <stdexcept>
47 #include <string>
48
49 namespace ROCKSDB_NAMESPACE {
50
51 class FileChecksumGenCrc32c;
52 class FileChecksumGenCrc32cFactory;
53
54 const std::string LDBCommand::ARG_ENV_URI = "env_uri";
55 const std::string LDBCommand::ARG_DB = "db";
56 const std::string LDBCommand::ARG_PATH = "path";
57 const std::string LDBCommand::ARG_SECONDARY_PATH = "secondary_path";
58 const std::string LDBCommand::ARG_HEX = "hex";
59 const std::string LDBCommand::ARG_KEY_HEX = "key_hex";
60 const std::string LDBCommand::ARG_VALUE_HEX = "value_hex";
61 const std::string LDBCommand::ARG_CF_NAME = "column_family";
62 const std::string LDBCommand::ARG_TTL = "ttl";
63 const std::string LDBCommand::ARG_TTL_START = "start_time";
64 const std::string LDBCommand::ARG_TTL_END = "end_time";
65 const std::string LDBCommand::ARG_TIMESTAMP = "timestamp";
66 const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options";
67 const std::string LDBCommand::ARG_IGNORE_UNKNOWN_OPTIONS =
68 "ignore_unknown_options";
69 const std::string LDBCommand::ARG_FROM = "from";
70 const std::string LDBCommand::ARG_TO = "to";
71 const std::string LDBCommand::ARG_MAX_KEYS = "max_keys";
72 const std::string LDBCommand::ARG_BLOOM_BITS = "bloom_bits";
73 const std::string LDBCommand::ARG_FIX_PREFIX_LEN = "fix_prefix_len";
74 const std::string LDBCommand::ARG_COMPRESSION_TYPE = "compression_type";
75 const std::string LDBCommand::ARG_COMPRESSION_MAX_DICT_BYTES =
76 "compression_max_dict_bytes";
77 const std::string LDBCommand::ARG_BLOCK_SIZE = "block_size";
78 const std::string LDBCommand::ARG_AUTO_COMPACTION = "auto_compaction";
79 const std::string LDBCommand::ARG_DB_WRITE_BUFFER_SIZE = "db_write_buffer_size";
80 const std::string LDBCommand::ARG_WRITE_BUFFER_SIZE = "write_buffer_size";
81 const std::string LDBCommand::ARG_FILE_SIZE = "file_size";
82 const std::string LDBCommand::ARG_CREATE_IF_MISSING = "create_if_missing";
83 const std::string LDBCommand::ARG_NO_VALUE = "no_value";
84
85 const char* LDBCommand::DELIM = " ==> ";
86
87 namespace {
88
89 void DumpWalFile(Options options, std::string wal_file, bool print_header,
90 bool print_values, bool is_write_committed,
91 LDBCommandExecuteResult* exec_state);
92
93 void DumpSstFile(Options options, std::string filename, bool output_hex,
94 bool show_properties);
95 };
96
InitFromCmdLineArgs(int argc,char ** argv,const Options & options,const LDBOptions & ldb_options,const std::vector<ColumnFamilyDescriptor> * column_families)97 LDBCommand* LDBCommand::InitFromCmdLineArgs(
98 int argc, char** argv, const Options& options,
99 const LDBOptions& ldb_options,
100 const std::vector<ColumnFamilyDescriptor>* column_families) {
101 std::vector<std::string> args;
102 for (int i = 1; i < argc; i++) {
103 args.push_back(argv[i]);
104 }
105 return InitFromCmdLineArgs(args, options, ldb_options, column_families,
106 SelectCommand);
107 }
108
109 /**
110 * Parse the command-line arguments and create the appropriate LDBCommand2
111 * instance.
112 * The command line arguments must be in the following format:
113 * ./ldb --db=PATH_TO_DB [--commonOpt1=commonOpt1Val] ..
114 * COMMAND <PARAM1> <PARAM2> ... [-cmdSpecificOpt1=cmdSpecificOpt1Val] ..
115 * This is similar to the command line format used by HBaseClientTool.
116 * Command name is not included in args.
117 * Returns nullptr if the command-line cannot be parsed.
118 */
InitFromCmdLineArgs(const std::vector<std::string> & args,const Options & options,const LDBOptions & ldb_options,const std::vector<ColumnFamilyDescriptor> *,const std::function<LDBCommand * (const ParsedParams &)> & selector)119 LDBCommand* LDBCommand::InitFromCmdLineArgs(
120 const std::vector<std::string>& args, const Options& options,
121 const LDBOptions& ldb_options,
122 const std::vector<ColumnFamilyDescriptor>* /*column_families*/,
123 const std::function<LDBCommand*(const ParsedParams&)>& selector) {
124 // --x=y command line arguments are added as x->y map entries in
125 // parsed_params.option_map.
126 //
127 // Command-line arguments of the form --hex end up in this array as hex to
128 // parsed_params.flags
129 ParsedParams parsed_params;
130
131 // Everything other than option_map and flags. Represents commands
132 // and their parameters. For eg: put key1 value1 go into this vector.
133 std::vector<std::string> cmdTokens;
134
135 const std::string OPTION_PREFIX = "--";
136
137 for (const auto& arg : args) {
138 if (arg[0] == '-' && arg[1] == '-'){
139 std::vector<std::string> splits = StringSplit(arg, '=');
140 // --option_name=option_value
141 if (splits.size() == 2) {
142 std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
143 parsed_params.option_map[optionKey] = splits[1];
144 } else if (splits.size() == 1) {
145 // --flag_name
146 std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
147 parsed_params.flags.push_back(optionKey);
148 } else {
149 // --option_name=option_value, option_value contains '='
150 std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
151 parsed_params.option_map[optionKey] =
152 arg.substr(splits[0].length() + 1);
153 }
154 } else {
155 cmdTokens.push_back(arg);
156 }
157 }
158
159 if (cmdTokens.size() < 1) {
160 fprintf(stderr, "Command not specified!");
161 return nullptr;
162 }
163
164 parsed_params.cmd = cmdTokens[0];
165 parsed_params.cmd_params.assign(cmdTokens.begin() + 1, cmdTokens.end());
166
167 LDBCommand* command = selector(parsed_params);
168
169 if (command) {
170 command->SetDBOptions(options);
171 command->SetLDBOptions(ldb_options);
172 }
173 return command;
174 }
175
SelectCommand(const ParsedParams & parsed_params)176 LDBCommand* LDBCommand::SelectCommand(const ParsedParams& parsed_params) {
177 if (parsed_params.cmd == GetCommand::Name()) {
178 return new GetCommand(parsed_params.cmd_params, parsed_params.option_map,
179 parsed_params.flags);
180 } else if (parsed_params.cmd == PutCommand::Name()) {
181 return new PutCommand(parsed_params.cmd_params, parsed_params.option_map,
182 parsed_params.flags);
183 } else if (parsed_params.cmd == BatchPutCommand::Name()) {
184 return new BatchPutCommand(parsed_params.cmd_params,
185 parsed_params.option_map, parsed_params.flags);
186 } else if (parsed_params.cmd == ScanCommand::Name()) {
187 return new ScanCommand(parsed_params.cmd_params, parsed_params.option_map,
188 parsed_params.flags);
189 } else if (parsed_params.cmd == DeleteCommand::Name()) {
190 return new DeleteCommand(parsed_params.cmd_params, parsed_params.option_map,
191 parsed_params.flags);
192 } else if (parsed_params.cmd == DeleteRangeCommand::Name()) {
193 return new DeleteRangeCommand(parsed_params.cmd_params,
194 parsed_params.option_map,
195 parsed_params.flags);
196 } else if (parsed_params.cmd == ApproxSizeCommand::Name()) {
197 return new ApproxSizeCommand(parsed_params.cmd_params,
198 parsed_params.option_map, parsed_params.flags);
199 } else if (parsed_params.cmd == DBQuerierCommand::Name()) {
200 return new DBQuerierCommand(parsed_params.cmd_params,
201 parsed_params.option_map, parsed_params.flags);
202 } else if (parsed_params.cmd == CompactorCommand::Name()) {
203 return new CompactorCommand(parsed_params.cmd_params,
204 parsed_params.option_map, parsed_params.flags);
205 } else if (parsed_params.cmd == WALDumperCommand::Name()) {
206 return new WALDumperCommand(parsed_params.cmd_params,
207 parsed_params.option_map, parsed_params.flags);
208 } else if (parsed_params.cmd == ReduceDBLevelsCommand::Name()) {
209 return new ReduceDBLevelsCommand(parsed_params.cmd_params,
210 parsed_params.option_map,
211 parsed_params.flags);
212 } else if (parsed_params.cmd == ChangeCompactionStyleCommand::Name()) {
213 return new ChangeCompactionStyleCommand(parsed_params.cmd_params,
214 parsed_params.option_map,
215 parsed_params.flags);
216 } else if (parsed_params.cmd == DBDumperCommand::Name()) {
217 return new DBDumperCommand(parsed_params.cmd_params,
218 parsed_params.option_map, parsed_params.flags);
219 } else if (parsed_params.cmd == DBLoaderCommand::Name()) {
220 return new DBLoaderCommand(parsed_params.cmd_params,
221 parsed_params.option_map, parsed_params.flags);
222 } else if (parsed_params.cmd == ManifestDumpCommand::Name()) {
223 return new ManifestDumpCommand(parsed_params.cmd_params,
224 parsed_params.option_map,
225 parsed_params.flags);
226 } else if (parsed_params.cmd == FileChecksumDumpCommand::Name()) {
227 return new FileChecksumDumpCommand(parsed_params.cmd_params,
228 parsed_params.option_map,
229 parsed_params.flags);
230 } else if (parsed_params.cmd == ListColumnFamiliesCommand::Name()) {
231 return new ListColumnFamiliesCommand(parsed_params.cmd_params,
232 parsed_params.option_map,
233 parsed_params.flags);
234 } else if (parsed_params.cmd == CreateColumnFamilyCommand::Name()) {
235 return new CreateColumnFamilyCommand(parsed_params.cmd_params,
236 parsed_params.option_map,
237 parsed_params.flags);
238 } else if (parsed_params.cmd == DropColumnFamilyCommand::Name()) {
239 return new DropColumnFamilyCommand(parsed_params.cmd_params,
240 parsed_params.option_map,
241 parsed_params.flags);
242 } else if (parsed_params.cmd == DBFileDumperCommand::Name()) {
243 return new DBFileDumperCommand(parsed_params.cmd_params,
244 parsed_params.option_map,
245 parsed_params.flags);
246 } else if (parsed_params.cmd == InternalDumpCommand::Name()) {
247 return new InternalDumpCommand(parsed_params.cmd_params,
248 parsed_params.option_map,
249 parsed_params.flags);
250 } else if (parsed_params.cmd == CheckConsistencyCommand::Name()) {
251 return new CheckConsistencyCommand(parsed_params.cmd_params,
252 parsed_params.option_map,
253 parsed_params.flags);
254 } else if (parsed_params.cmd == CheckPointCommand::Name()) {
255 return new CheckPointCommand(parsed_params.cmd_params,
256 parsed_params.option_map,
257 parsed_params.flags);
258 } else if (parsed_params.cmd == RepairCommand::Name()) {
259 return new RepairCommand(parsed_params.cmd_params, parsed_params.option_map,
260 parsed_params.flags);
261 } else if (parsed_params.cmd == BackupCommand::Name()) {
262 return new BackupCommand(parsed_params.cmd_params, parsed_params.option_map,
263 parsed_params.flags);
264 } else if (parsed_params.cmd == RestoreCommand::Name()) {
265 return new RestoreCommand(parsed_params.cmd_params,
266 parsed_params.option_map, parsed_params.flags);
267 } else if (parsed_params.cmd == WriteExternalSstFilesCommand::Name()) {
268 return new WriteExternalSstFilesCommand(parsed_params.cmd_params,
269 parsed_params.option_map,
270 parsed_params.flags);
271 } else if (parsed_params.cmd == IngestExternalSstFilesCommand::Name()) {
272 return new IngestExternalSstFilesCommand(parsed_params.cmd_params,
273 parsed_params.option_map,
274 parsed_params.flags);
275 } else if (parsed_params.cmd == ListFileRangeDeletesCommand::Name()) {
276 return new ListFileRangeDeletesCommand(parsed_params.option_map,
277 parsed_params.flags);
278 }
279 return nullptr;
280 }
281
282 /* Run the command, and return the execute result. */
Run()283 void LDBCommand::Run() {
284 if (!exec_state_.IsNotStarted()) {
285 return;
286 }
287
288 if (!options_.env || options_.env == Env::Default()) {
289 Env* env = Env::Default();
290 Status s = Env::LoadEnv(env_uri_, &env, &env_guard_);
291 if (!s.ok() && !s.IsNotFound()) {
292 fprintf(stderr, "LoadEnv: %s\n", s.ToString().c_str());
293 exec_state_ = LDBCommandExecuteResult::Failed(s.ToString());
294 return;
295 }
296 options_.env = env;
297 }
298
299 if (db_ == nullptr && !NoDBOpen()) {
300 OpenDB();
301 if (exec_state_.IsFailed() && try_load_options_) {
302 // We don't always return if there is a failure because a WAL file or
303 // manifest file can be given to "dump" command so we should continue.
304 // --try_load_options is not valid in those cases.
305 return;
306 }
307 }
308
309 // We'll intentionally proceed even if the DB can't be opened because users
310 // can also specify a filename, not just a directory.
311 DoCommand();
312
313 if (exec_state_.IsNotStarted()) {
314 exec_state_ = LDBCommandExecuteResult::Succeed("");
315 }
316
317 if (db_ != nullptr) {
318 CloseDB();
319 }
320 }
321
LDBCommand(const std::map<std::string,std::string> & options,const std::vector<std::string> & flags,bool is_read_only,const std::vector<std::string> & valid_cmd_line_options)322 LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
323 const std::vector<std::string>& flags, bool is_read_only,
324 const std::vector<std::string>& valid_cmd_line_options)
325 : db_(nullptr),
326 db_ttl_(nullptr),
327 is_read_only_(is_read_only),
328 is_key_hex_(false),
329 is_value_hex_(false),
330 is_db_ttl_(false),
331 timestamp_(false),
332 try_load_options_(false),
333 ignore_unknown_options_(false),
334 create_if_missing_(false),
335 option_map_(options),
336 flags_(flags),
337 valid_cmd_line_options_(valid_cmd_line_options) {
338 std::map<std::string, std::string>::const_iterator itr = options.find(ARG_DB);
339 if (itr != options.end()) {
340 db_path_ = itr->second;
341 }
342
343 itr = options.find(ARG_ENV_URI);
344 if (itr != options.end()) {
345 env_uri_ = itr->second;
346 }
347
348 itr = options.find(ARG_CF_NAME);
349 if (itr != options.end()) {
350 column_family_name_ = itr->second;
351 } else {
352 column_family_name_ = kDefaultColumnFamilyName;
353 }
354
355 itr = options.find(ARG_SECONDARY_PATH);
356 secondary_path_ = "";
357 if (itr != options.end()) {
358 secondary_path_ = itr->second;
359 }
360
361 is_key_hex_ = IsKeyHex(options, flags);
362 is_value_hex_ = IsValueHex(options, flags);
363 is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
364 timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP);
365 try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS);
366 ignore_unknown_options_ = IsFlagPresent(flags, ARG_IGNORE_UNKNOWN_OPTIONS);
367 }
368
OpenDB()369 void LDBCommand::OpenDB() {
370 if (!create_if_missing_ && try_load_options_) {
371 Status s = LoadLatestOptions(db_path_, options_.env, &options_,
372 &column_families_, ignore_unknown_options_);
373 if (!s.ok() && !s.IsNotFound()) {
374 // Option file exists but load option file error.
375 std::string msg = s.ToString();
376 exec_state_ = LDBCommandExecuteResult::Failed(msg);
377 db_ = nullptr;
378 return;
379 }
380 if (options_.env->FileExists(options_.wal_dir).IsNotFound()) {
381 options_.wal_dir = db_path_;
382 fprintf(
383 stderr,
384 "wal_dir loaded from the option file doesn't exist. Ignore it.\n");
385 }
386
387 // If merge operator is not set, set a string append operator. There is
388 // no harm doing it.
389 for (auto& cf_entry : column_families_) {
390 if (!cf_entry.options.merge_operator) {
391 cf_entry.options.merge_operator =
392 MergeOperators::CreateStringAppendOperator(':');
393 }
394 }
395 }
396 options_ = PrepareOptionsForOpenDB();
397 if (!exec_state_.IsNotStarted()) {
398 return;
399 }
400 if (column_families_.empty() && !options_.merge_operator) {
401 // No harm to add a general merge operator if it is not specified.
402 options_.merge_operator = MergeOperators::CreateStringAppendOperator(':');
403 }
404 // Open the DB.
405 Status st;
406 std::vector<ColumnFamilyHandle*> handles_opened;
407 if (is_db_ttl_) {
408 // ldb doesn't yet support TTL DB with multiple column families
409 if (!column_family_name_.empty() || !column_families_.empty()) {
410 exec_state_ = LDBCommandExecuteResult::Failed(
411 "ldb doesn't support TTL DB with multiple column families");
412 }
413 if (!secondary_path_.empty()) {
414 exec_state_ = LDBCommandExecuteResult::Failed(
415 "Open as secondary is not supported for TTL DB yet.");
416 }
417 if (is_read_only_) {
418 st = DBWithTTL::Open(options_, db_path_, &db_ttl_, 0, true);
419 } else {
420 st = DBWithTTL::Open(options_, db_path_, &db_ttl_);
421 }
422 db_ = db_ttl_;
423 } else {
424 if (column_families_.empty()) {
425 // Try to figure out column family lists
426 std::vector<std::string> cf_list;
427 st = DB::ListColumnFamilies(options_, db_path_, &cf_list);
428 // There is possible the DB doesn't exist yet, for "create if not
429 // "existing case". The failure is ignored here. We rely on DB::Open()
430 // to give us the correct error message for problem with opening
431 // existing DB.
432 if (st.ok() && cf_list.size() > 1) {
433 // Ignore single column family DB.
434 for (auto cf_name : cf_list) {
435 column_families_.emplace_back(cf_name, options_);
436 }
437 }
438 }
439 if (is_read_only_ && secondary_path_.empty()) {
440 if (column_families_.empty()) {
441 st = DB::OpenForReadOnly(options_, db_path_, &db_);
442 } else {
443 st = DB::OpenForReadOnly(options_, db_path_, column_families_,
444 &handles_opened, &db_);
445 }
446 } else {
447 if (column_families_.empty()) {
448 if (secondary_path_.empty()) {
449 st = DB::Open(options_, db_path_, &db_);
450 } else {
451 st = DB::OpenAsSecondary(options_, db_path_, secondary_path_, &db_);
452 }
453 } else {
454 if (secondary_path_.empty()) {
455 st = DB::Open(options_, db_path_, column_families_, &handles_opened,
456 &db_);
457 } else {
458 st = DB::OpenAsSecondary(options_, db_path_, secondary_path_,
459 column_families_, &handles_opened, &db_);
460 }
461 }
462 }
463 }
464 if (!st.ok()) {
465 std::string msg = st.ToString();
466 exec_state_ = LDBCommandExecuteResult::Failed(msg);
467 } else if (!handles_opened.empty()) {
468 assert(handles_opened.size() == column_families_.size());
469 bool found_cf_name = false;
470 for (size_t i = 0; i < handles_opened.size(); i++) {
471 cf_handles_[column_families_[i].name] = handles_opened[i];
472 if (column_family_name_ == column_families_[i].name) {
473 found_cf_name = true;
474 }
475 }
476 if (!found_cf_name) {
477 exec_state_ = LDBCommandExecuteResult::Failed(
478 "Non-existing column family " + column_family_name_);
479 CloseDB();
480 }
481 } else {
482 // We successfully opened DB in single column family mode.
483 assert(column_families_.empty());
484 if (column_family_name_ != kDefaultColumnFamilyName) {
485 exec_state_ = LDBCommandExecuteResult::Failed(
486 "Non-existing column family " + column_family_name_);
487 CloseDB();
488 }
489 }
490 }
491
CloseDB()492 void LDBCommand::CloseDB() {
493 if (db_ != nullptr) {
494 for (auto& pair : cf_handles_) {
495 delete pair.second;
496 }
497 delete db_;
498 db_ = nullptr;
499 }
500 }
501
GetCfHandle()502 ColumnFamilyHandle* LDBCommand::GetCfHandle() {
503 if (!cf_handles_.empty()) {
504 auto it = cf_handles_.find(column_family_name_);
505 if (it == cf_handles_.end()) {
506 exec_state_ = LDBCommandExecuteResult::Failed(
507 "Cannot find column family " + column_family_name_);
508 } else {
509 return it->second;
510 }
511 }
512 return db_->DefaultColumnFamily();
513 }
514
BuildCmdLineOptions(std::vector<std::string> options)515 std::vector<std::string> LDBCommand::BuildCmdLineOptions(
516 std::vector<std::string> options) {
517 std::vector<std::string> ret = {ARG_ENV_URI,
518 ARG_DB,
519 ARG_SECONDARY_PATH,
520 ARG_BLOOM_BITS,
521 ARG_BLOCK_SIZE,
522 ARG_AUTO_COMPACTION,
523 ARG_COMPRESSION_TYPE,
524 ARG_COMPRESSION_MAX_DICT_BYTES,
525 ARG_WRITE_BUFFER_SIZE,
526 ARG_FILE_SIZE,
527 ARG_FIX_PREFIX_LEN,
528 ARG_TRY_LOAD_OPTIONS,
529 ARG_IGNORE_UNKNOWN_OPTIONS,
530 ARG_CF_NAME};
531 ret.insert(ret.end(), options.begin(), options.end());
532 return ret;
533 }
534
535 /**
536 * Parses the specific integer option and fills in the value.
537 * Returns true if the option is found.
538 * Returns false if the option is not found or if there is an error parsing the
539 * value. If there is an error, the specified exec_state is also
540 * updated.
541 */
ParseIntOption(const std::map<std::string,std::string> &,const std::string & option,int & value,LDBCommandExecuteResult & exec_state)542 bool LDBCommand::ParseIntOption(
543 const std::map<std::string, std::string>& /*options*/,
544 const std::string& option, int& value,
545 LDBCommandExecuteResult& exec_state) {
546 std::map<std::string, std::string>::const_iterator itr =
547 option_map_.find(option);
548 if (itr != option_map_.end()) {
549 try {
550 #if defined(CYGWIN)
551 value = strtol(itr->second.c_str(), 0, 10);
552 #else
553 value = std::stoi(itr->second);
554 #endif
555 return true;
556 } catch (const std::invalid_argument&) {
557 exec_state =
558 LDBCommandExecuteResult::Failed(option + " has an invalid value.");
559 } catch (const std::out_of_range&) {
560 exec_state = LDBCommandExecuteResult::Failed(
561 option + " has a value out-of-range.");
562 }
563 }
564 return false;
565 }
566
567 /**
568 * Parses the specified option and fills in the value.
569 * Returns true if the option is found.
570 * Returns false otherwise.
571 */
ParseStringOption(const std::map<std::string,std::string> &,const std::string & option,std::string * value)572 bool LDBCommand::ParseStringOption(
573 const std::map<std::string, std::string>& /*options*/,
574 const std::string& option, std::string* value) {
575 auto itr = option_map_.find(option);
576 if (itr != option_map_.end()) {
577 *value = itr->second;
578 return true;
579 }
580 return false;
581 }
582
PrepareOptionsForOpenDB()583 Options LDBCommand::PrepareOptionsForOpenDB() {
584 ColumnFamilyOptions* cf_opts;
585 auto column_families_iter =
586 std::find_if(column_families_.begin(), column_families_.end(),
587 [this](const ColumnFamilyDescriptor& cf_desc) {
588 return cf_desc.name == column_family_name_;
589 });
590 if (column_families_iter != column_families_.end()) {
591 cf_opts = &column_families_iter->options;
592 } else {
593 cf_opts = static_cast<ColumnFamilyOptions*>(&options_);
594 }
595 DBOptions* db_opts = static_cast<DBOptions*>(&options_);
596 db_opts->create_if_missing = false;
597
598 std::map<std::string, std::string>::const_iterator itr;
599
600 BlockBasedTableOptions table_options;
601 bool use_table_options = false;
602 int bits;
603 if (ParseIntOption(option_map_, ARG_BLOOM_BITS, bits, exec_state_)) {
604 if (bits > 0) {
605 use_table_options = true;
606 table_options.filter_policy.reset(NewBloomFilterPolicy(bits));
607 } else {
608 exec_state_ =
609 LDBCommandExecuteResult::Failed(ARG_BLOOM_BITS + " must be > 0.");
610 }
611 }
612
613 int block_size;
614 if (ParseIntOption(option_map_, ARG_BLOCK_SIZE, block_size, exec_state_)) {
615 if (block_size > 0) {
616 use_table_options = true;
617 table_options.block_size = block_size;
618 } else {
619 exec_state_ =
620 LDBCommandExecuteResult::Failed(ARG_BLOCK_SIZE + " must be > 0.");
621 }
622 }
623
624 if (use_table_options) {
625 cf_opts->table_factory.reset(NewBlockBasedTableFactory(table_options));
626 }
627
628 itr = option_map_.find(ARG_AUTO_COMPACTION);
629 if (itr != option_map_.end()) {
630 cf_opts->disable_auto_compactions = !StringToBool(itr->second);
631 }
632
633 itr = option_map_.find(ARG_COMPRESSION_TYPE);
634 if (itr != option_map_.end()) {
635 std::string comp = itr->second;
636 if (comp == "no") {
637 cf_opts->compression = kNoCompression;
638 } else if (comp == "snappy") {
639 cf_opts->compression = kSnappyCompression;
640 } else if (comp == "zlib") {
641 cf_opts->compression = kZlibCompression;
642 } else if (comp == "bzip2") {
643 cf_opts->compression = kBZip2Compression;
644 } else if (comp == "lz4") {
645 cf_opts->compression = kLZ4Compression;
646 } else if (comp == "lz4hc") {
647 cf_opts->compression = kLZ4HCCompression;
648 } else if (comp == "xpress") {
649 cf_opts->compression = kXpressCompression;
650 } else if (comp == "zstd") {
651 cf_opts->compression = kZSTD;
652 } else {
653 // Unknown compression.
654 exec_state_ =
655 LDBCommandExecuteResult::Failed("Unknown compression level: " + comp);
656 }
657 }
658
659 int compression_max_dict_bytes;
660 if (ParseIntOption(option_map_, ARG_COMPRESSION_MAX_DICT_BYTES,
661 compression_max_dict_bytes, exec_state_)) {
662 if (compression_max_dict_bytes >= 0) {
663 cf_opts->compression_opts.max_dict_bytes = compression_max_dict_bytes;
664 } else {
665 exec_state_ = LDBCommandExecuteResult::Failed(
666 ARG_COMPRESSION_MAX_DICT_BYTES + " must be >= 0.");
667 }
668 }
669
670 int db_write_buffer_size;
671 if (ParseIntOption(option_map_, ARG_DB_WRITE_BUFFER_SIZE,
672 db_write_buffer_size, exec_state_)) {
673 if (db_write_buffer_size >= 0) {
674 db_opts->db_write_buffer_size = db_write_buffer_size;
675 } else {
676 exec_state_ = LDBCommandExecuteResult::Failed(ARG_DB_WRITE_BUFFER_SIZE +
677 " must be >= 0.");
678 }
679 }
680
681 int write_buffer_size;
682 if (ParseIntOption(option_map_, ARG_WRITE_BUFFER_SIZE, write_buffer_size,
683 exec_state_)) {
684 if (write_buffer_size > 0) {
685 cf_opts->write_buffer_size = write_buffer_size;
686 } else {
687 exec_state_ = LDBCommandExecuteResult::Failed(ARG_WRITE_BUFFER_SIZE +
688 " must be > 0.");
689 }
690 }
691
692 int file_size;
693 if (ParseIntOption(option_map_, ARG_FILE_SIZE, file_size, exec_state_)) {
694 if (file_size > 0) {
695 cf_opts->target_file_size_base = file_size;
696 } else {
697 exec_state_ =
698 LDBCommandExecuteResult::Failed(ARG_FILE_SIZE + " must be > 0.");
699 }
700 }
701
702 if (db_opts->db_paths.size() == 0) {
703 db_opts->db_paths.emplace_back(db_path_,
704 std::numeric_limits<uint64_t>::max());
705 }
706
707 int fix_prefix_len;
708 if (ParseIntOption(option_map_, ARG_FIX_PREFIX_LEN, fix_prefix_len,
709 exec_state_)) {
710 if (fix_prefix_len > 0) {
711 cf_opts->prefix_extractor.reset(
712 NewFixedPrefixTransform(static_cast<size_t>(fix_prefix_len)));
713 } else {
714 exec_state_ =
715 LDBCommandExecuteResult::Failed(ARG_FIX_PREFIX_LEN + " must be > 0.");
716 }
717 }
718
719 // TODO(ajkr): this return value doesn't reflect the CF options changed, so
720 // subcommands that rely on this won't see the effect of CF-related CLI args.
721 // Such subcommands need to be changed to properly support CFs.
722 return options_;
723 }
724
ParseKeyValue(const std::string & line,std::string * key,std::string * value,bool is_key_hex,bool is_value_hex)725 bool LDBCommand::ParseKeyValue(const std::string& line, std::string* key,
726 std::string* value, bool is_key_hex,
727 bool is_value_hex) {
728 size_t pos = line.find(DELIM);
729 if (pos != std::string::npos) {
730 *key = line.substr(0, pos);
731 *value = line.substr(pos + strlen(DELIM));
732 if (is_key_hex) {
733 *key = HexToString(*key);
734 }
735 if (is_value_hex) {
736 *value = HexToString(*value);
737 }
738 return true;
739 } else {
740 return false;
741 }
742 }
743
744 /**
745 * Make sure that ONLY the command-line options and flags expected by this
746 * command are specified on the command-line. Extraneous options are usually
747 * the result of user error.
748 * Returns true if all checks pass. Else returns false, and prints an
749 * appropriate error msg to stderr.
750 */
ValidateCmdLineOptions()751 bool LDBCommand::ValidateCmdLineOptions() {
752 for (std::map<std::string, std::string>::const_iterator itr =
753 option_map_.begin();
754 itr != option_map_.end(); ++itr) {
755 if (std::find(valid_cmd_line_options_.begin(),
756 valid_cmd_line_options_.end(),
757 itr->first) == valid_cmd_line_options_.end()) {
758 fprintf(stderr, "Invalid command-line option %s\n", itr->first.c_str());
759 return false;
760 }
761 }
762
763 for (std::vector<std::string>::const_iterator itr = flags_.begin();
764 itr != flags_.end(); ++itr) {
765 if (std::find(valid_cmd_line_options_.begin(),
766 valid_cmd_line_options_.end(),
767 *itr) == valid_cmd_line_options_.end()) {
768 fprintf(stderr, "Invalid command-line flag %s\n", itr->c_str());
769 return false;
770 }
771 }
772
773 if (!NoDBOpen() && option_map_.find(ARG_DB) == option_map_.end() &&
774 option_map_.find(ARG_PATH) == option_map_.end()) {
775 fprintf(stderr, "Either %s or %s must be specified.\n", ARG_DB.c_str(),
776 ARG_PATH.c_str());
777 return false;
778 }
779
780 return true;
781 }
782
HexToString(const std::string & str)783 std::string LDBCommand::HexToString(const std::string& str) {
784 std::string result;
785 std::string::size_type len = str.length();
786 if (len < 2 || str[0] != '0' || str[1] != 'x') {
787 fprintf(stderr, "Invalid hex input %s. Must start with 0x\n", str.c_str());
788 throw "Invalid hex input";
789 }
790 if (!Slice(str.data() + 2, len - 2).DecodeHex(&result)) {
791 throw "Invalid hex input";
792 }
793 return result;
794 }
795
StringToHex(const std::string & str)796 std::string LDBCommand::StringToHex(const std::string& str) {
797 std::string result("0x");
798 result.append(Slice(str).ToString(true));
799 return result;
800 }
801
PrintKeyValue(const std::string & key,const std::string & value,bool is_key_hex,bool is_value_hex)802 std::string LDBCommand::PrintKeyValue(const std::string& key,
803 const std::string& value, bool is_key_hex,
804 bool is_value_hex) {
805 std::string result;
806 result.append(is_key_hex ? StringToHex(key) : key);
807 result.append(DELIM);
808 result.append(is_value_hex ? StringToHex(value) : value);
809 return result;
810 }
811
PrintKeyValue(const std::string & key,const std::string & value,bool is_hex)812 std::string LDBCommand::PrintKeyValue(const std::string& key,
813 const std::string& value, bool is_hex) {
814 return PrintKeyValue(key, value, is_hex, is_hex);
815 }
816
HelpRangeCmdArgs()817 std::string LDBCommand::HelpRangeCmdArgs() {
818 std::ostringstream str_stream;
819 str_stream << " ";
820 str_stream << "[--" << ARG_FROM << "] ";
821 str_stream << "[--" << ARG_TO << "] ";
822 return str_stream.str();
823 }
824
IsKeyHex(const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)825 bool LDBCommand::IsKeyHex(const std::map<std::string, std::string>& options,
826 const std::vector<std::string>& flags) {
827 return (IsFlagPresent(flags, ARG_HEX) || IsFlagPresent(flags, ARG_KEY_HEX) ||
828 ParseBooleanOption(options, ARG_HEX, false) ||
829 ParseBooleanOption(options, ARG_KEY_HEX, false));
830 }
831
IsValueHex(const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)832 bool LDBCommand::IsValueHex(const std::map<std::string, std::string>& options,
833 const std::vector<std::string>& flags) {
834 return (IsFlagPresent(flags, ARG_HEX) ||
835 IsFlagPresent(flags, ARG_VALUE_HEX) ||
836 ParseBooleanOption(options, ARG_HEX, false) ||
837 ParseBooleanOption(options, ARG_VALUE_HEX, false));
838 }
839
ParseBooleanOption(const std::map<std::string,std::string> & options,const std::string & option,bool default_val)840 bool LDBCommand::ParseBooleanOption(
841 const std::map<std::string, std::string>& options,
842 const std::string& option, bool default_val) {
843 std::map<std::string, std::string>::const_iterator itr = options.find(option);
844 if (itr != options.end()) {
845 std::string option_val = itr->second;
846 return StringToBool(itr->second);
847 }
848 return default_val;
849 }
850
StringToBool(std::string val)851 bool LDBCommand::StringToBool(std::string val) {
852 std::transform(val.begin(), val.end(), val.begin(),
853 [](char ch) -> char { return (char)::tolower(ch); });
854
855 if (val == "true") {
856 return true;
857 } else if (val == "false") {
858 return false;
859 } else {
860 throw "Invalid value for boolean argument";
861 }
862 }
863
CompactorCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)864 CompactorCommand::CompactorCommand(
865 const std::vector<std::string>& /*params*/,
866 const std::map<std::string, std::string>& options,
867 const std::vector<std::string>& flags)
868 : LDBCommand(options, flags, false,
869 BuildCmdLineOptions({ARG_FROM, ARG_TO, ARG_HEX, ARG_KEY_HEX,
870 ARG_VALUE_HEX, ARG_TTL})),
871 null_from_(true),
872 null_to_(true) {
873 std::map<std::string, std::string>::const_iterator itr =
874 options.find(ARG_FROM);
875 if (itr != options.end()) {
876 null_from_ = false;
877 from_ = itr->second;
878 }
879
880 itr = options.find(ARG_TO);
881 if (itr != options.end()) {
882 null_to_ = false;
883 to_ = itr->second;
884 }
885
886 if (is_key_hex_) {
887 if (!null_from_) {
888 from_ = HexToString(from_);
889 }
890 if (!null_to_) {
891 to_ = HexToString(to_);
892 }
893 }
894 }
895
Help(std::string & ret)896 void CompactorCommand::Help(std::string& ret) {
897 ret.append(" ");
898 ret.append(CompactorCommand::Name());
899 ret.append(HelpRangeCmdArgs());
900 ret.append("\n");
901 }
902
DoCommand()903 void CompactorCommand::DoCommand() {
904 if (!db_) {
905 assert(GetExecuteState().IsFailed());
906 return;
907 }
908
909 Slice* begin = nullptr;
910 Slice* end = nullptr;
911 if (!null_from_) {
912 begin = new Slice(from_);
913 }
914 if (!null_to_) {
915 end = new Slice(to_);
916 }
917
918 CompactRangeOptions cro;
919 cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized;
920
921 db_->CompactRange(cro, GetCfHandle(), begin, end);
922 exec_state_ = LDBCommandExecuteResult::Succeed("");
923
924 delete begin;
925 delete end;
926 }
927
928 // ---------------------------------------------------------------------------
929 const std::string DBLoaderCommand::ARG_DISABLE_WAL = "disable_wal";
930 const std::string DBLoaderCommand::ARG_BULK_LOAD = "bulk_load";
931 const std::string DBLoaderCommand::ARG_COMPACT = "compact";
932
DBLoaderCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)933 DBLoaderCommand::DBLoaderCommand(
934 const std::vector<std::string>& /*params*/,
935 const std::map<std::string, std::string>& options,
936 const std::vector<std::string>& flags)
937 : LDBCommand(
938 options, flags, false,
939 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
940 ARG_TO, ARG_CREATE_IF_MISSING, ARG_DISABLE_WAL,
941 ARG_BULK_LOAD, ARG_COMPACT})),
942 disable_wal_(false),
943 bulk_load_(false),
944 compact_(false) {
945 create_if_missing_ = IsFlagPresent(flags, ARG_CREATE_IF_MISSING);
946 disable_wal_ = IsFlagPresent(flags, ARG_DISABLE_WAL);
947 bulk_load_ = IsFlagPresent(flags, ARG_BULK_LOAD);
948 compact_ = IsFlagPresent(flags, ARG_COMPACT);
949 }
950
Help(std::string & ret)951 void DBLoaderCommand::Help(std::string& ret) {
952 ret.append(" ");
953 ret.append(DBLoaderCommand::Name());
954 ret.append(" [--" + ARG_CREATE_IF_MISSING + "]");
955 ret.append(" [--" + ARG_DISABLE_WAL + "]");
956 ret.append(" [--" + ARG_BULK_LOAD + "]");
957 ret.append(" [--" + ARG_COMPACT + "]");
958 ret.append("\n");
959 }
960
PrepareOptionsForOpenDB()961 Options DBLoaderCommand::PrepareOptionsForOpenDB() {
962 Options opt = LDBCommand::PrepareOptionsForOpenDB();
963 opt.create_if_missing = create_if_missing_;
964 if (bulk_load_) {
965 opt.PrepareForBulkLoad();
966 }
967 return opt;
968 }
969
DoCommand()970 void DBLoaderCommand::DoCommand() {
971 if (!db_) {
972 assert(GetExecuteState().IsFailed());
973 return;
974 }
975
976 WriteOptions write_options;
977 if (disable_wal_) {
978 write_options.disableWAL = true;
979 }
980
981 int bad_lines = 0;
982 std::string line;
983 // prefer ifstream getline performance vs that from std::cin istream
984 std::ifstream ifs_stdin("/dev/stdin");
985 std::istream* istream_p = ifs_stdin.is_open() ? &ifs_stdin : &std::cin;
986 while (getline(*istream_p, line, '\n')) {
987 std::string key;
988 std::string value;
989 if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) {
990 db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value));
991 } else if (0 == line.find("Keys in range:")) {
992 // ignore this line
993 } else if (0 == line.find("Created bg thread 0x")) {
994 // ignore this line
995 } else {
996 bad_lines ++;
997 }
998 }
999
1000 if (bad_lines > 0) {
1001 std::cout << "Warning: " << bad_lines << " bad lines ignored." << std::endl;
1002 }
1003 if (compact_) {
1004 db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr);
1005 }
1006 }
1007
1008 // ----------------------------------------------------------------------------
1009
1010 namespace {
1011
DumpManifestFile(Options options,std::string file,bool verbose,bool hex,bool json)1012 void DumpManifestFile(Options options, std::string file, bool verbose, bool hex,
1013 bool json) {
1014 EnvOptions sopt;
1015 std::string dbname("dummy");
1016 std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10,
1017 options.table_cache_numshardbits));
1018 // Notice we are using the default options not through SanitizeOptions(),
1019 // if VersionSet::DumpManifest() depends on any option done by
1020 // SanitizeOptions(), we need to initialize it manually.
1021 options.db_paths.emplace_back("dummy", 0);
1022 options.num_levels = 64;
1023 WriteController wc(options.delayed_write_rate);
1024 WriteBufferManager wb(options.db_write_buffer_size);
1025 ImmutableDBOptions immutable_db_options(options);
1026 VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc,
1027 /*block_cache_tracer=*/nullptr);
1028 Status s = versions.DumpManifest(options, file, verbose, hex, json);
1029 if (!s.ok()) {
1030 fprintf(stderr, "Error in processing file %s %s\n", file.c_str(),
1031 s.ToString().c_str());
1032 }
1033 }
1034
1035 } // namespace
1036
1037 const std::string ManifestDumpCommand::ARG_VERBOSE = "verbose";
1038 const std::string ManifestDumpCommand::ARG_JSON = "json";
1039 const std::string ManifestDumpCommand::ARG_PATH = "path";
1040
Help(std::string & ret)1041 void ManifestDumpCommand::Help(std::string& ret) {
1042 ret.append(" ");
1043 ret.append(ManifestDumpCommand::Name());
1044 ret.append(" [--" + ARG_VERBOSE + "]");
1045 ret.append(" [--" + ARG_JSON + "]");
1046 ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]");
1047 ret.append("\n");
1048 }
1049
ManifestDumpCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)1050 ManifestDumpCommand::ManifestDumpCommand(
1051 const std::vector<std::string>& /*params*/,
1052 const std::map<std::string, std::string>& options,
1053 const std::vector<std::string>& flags)
1054 : LDBCommand(
1055 options, flags, false,
1056 BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX, ARG_JSON})),
1057 verbose_(false),
1058 json_(false),
1059 path_("") {
1060 verbose_ = IsFlagPresent(flags, ARG_VERBOSE);
1061 json_ = IsFlagPresent(flags, ARG_JSON);
1062
1063 std::map<std::string, std::string>::const_iterator itr =
1064 options.find(ARG_PATH);
1065 if (itr != options.end()) {
1066 path_ = itr->second;
1067 if (path_.empty()) {
1068 exec_state_ = LDBCommandExecuteResult::Failed("--path: missing pathname");
1069 }
1070 }
1071 }
1072
DoCommand()1073 void ManifestDumpCommand::DoCommand() {
1074
1075 std::string manifestfile;
1076
1077 if (!path_.empty()) {
1078 manifestfile = path_;
1079 } else {
1080 // We need to find the manifest file by searching the directory
1081 // containing the db for files of the form MANIFEST_[0-9]+
1082
1083 std::vector<std::string> files;
1084 Status s = options_.env->GetChildren(db_path_, &files);
1085 if (!s.ok()) {
1086 std::string err_msg = s.ToString();
1087 err_msg.append(": Failed to list the content of ");
1088 err_msg.append(db_path_);
1089 exec_state_ = LDBCommandExecuteResult::Failed(err_msg);
1090 return;
1091 }
1092 const std::string kManifestNamePrefix = "MANIFEST-";
1093 std::string matched_file;
1094 #ifdef OS_WIN
1095 const char kPathDelim = '\\';
1096 #else
1097 const char kPathDelim = '/';
1098 #endif
1099 for (const auto& file_path : files) {
1100 // Some Env::GetChildren() return absolute paths. Some directories' path
1101 // end with path delim, e.g. '/' or '\\'.
1102 size_t pos = file_path.find_last_of(kPathDelim);
1103 if (pos == file_path.size() - 1) {
1104 continue;
1105 }
1106 std::string fname;
1107 if (pos != std::string::npos) {
1108 // Absolute path.
1109 fname.assign(file_path, pos + 1, file_path.size() - pos - 1);
1110 } else {
1111 fname = file_path;
1112 }
1113 uint64_t file_num = 0;
1114 FileType file_type = kLogFile; // Just for initialization
1115 if (ParseFileName(fname, &file_num, &file_type) &&
1116 file_type == kDescriptorFile) {
1117 if (!matched_file.empty()) {
1118 exec_state_ = LDBCommandExecuteResult::Failed(
1119 "Multiple MANIFEST files found; use --path to select one");
1120 return;
1121 } else {
1122 matched_file.swap(fname);
1123 }
1124 }
1125 }
1126 if (matched_file.empty()) {
1127 std::string err_msg("No MANIFEST found in ");
1128 err_msg.append(db_path_);
1129 exec_state_ = LDBCommandExecuteResult::Failed(err_msg);
1130 return;
1131 }
1132 if (db_path_[db_path_.length() - 1] != '/') {
1133 db_path_.append("/");
1134 }
1135 manifestfile = db_path_ + matched_file;
1136 }
1137
1138 if (verbose_) {
1139 fprintf(stdout, "Processing Manifest file %s\n", manifestfile.c_str());
1140 }
1141
1142 DumpManifestFile(options_, manifestfile, verbose_, is_key_hex_, json_);
1143
1144 if (verbose_) {
1145 fprintf(stdout, "Processing Manifest file %s done\n", manifestfile.c_str());
1146 }
1147 }
1148
1149 // ----------------------------------------------------------------------------
1150 namespace {
1151
GetLiveFilesChecksumInfoFromVersionSet(Options options,const std::string & db_path,FileChecksumList * checksum_list)1152 void GetLiveFilesChecksumInfoFromVersionSet(Options options,
1153 const std::string& db_path,
1154 FileChecksumList* checksum_list) {
1155 EnvOptions sopt;
1156 Status s;
1157 std::string dbname(db_path);
1158 std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10,
1159 options.table_cache_numshardbits));
1160 // Notice we are using the default options not through SanitizeOptions(),
1161 // if VersionSet::GetLiveFilesChecksumInfo depends on any option done by
1162 // SanitizeOptions(), we need to initialize it manually.
1163 options.db_paths.emplace_back(db_path, 0);
1164 options.num_levels = 64;
1165 WriteController wc(options.delayed_write_rate);
1166 WriteBufferManager wb(options.db_write_buffer_size);
1167 ImmutableDBOptions immutable_db_options(options);
1168 VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc,
1169 /*block_cache_tracer=*/nullptr);
1170 std::vector<std::string> cf_name_list;
1171 s = versions.ListColumnFamilies(&cf_name_list, db_path,
1172 immutable_db_options.fs.get());
1173 if (s.ok()) {
1174 std::vector<ColumnFamilyDescriptor> cf_list;
1175 for (const auto& name : cf_name_list) {
1176 cf_list.emplace_back(name, ColumnFamilyOptions(options));
1177 }
1178 s = versions.Recover(cf_list, true);
1179 }
1180 if (s.ok()) {
1181 s = versions.GetLiveFilesChecksumInfo(checksum_list);
1182 }
1183 if (!s.ok()) {
1184 fprintf(stderr, "Error Status: %s", s.ToString().c_str());
1185 }
1186 }
1187
1188 } // namespace
1189
1190 const std::string FileChecksumDumpCommand::ARG_PATH = "path";
1191
Help(std::string & ret)1192 void FileChecksumDumpCommand::Help(std::string& ret) {
1193 ret.append(" ");
1194 ret.append(FileChecksumDumpCommand::Name());
1195 ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]");
1196 ret.append("\n");
1197 }
1198
FileChecksumDumpCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)1199 FileChecksumDumpCommand::FileChecksumDumpCommand(
1200 const std::vector<std::string>& /*params*/,
1201 const std::map<std::string, std::string>& options,
1202 const std::vector<std::string>& flags)
1203 : LDBCommand(options, flags, false, BuildCmdLineOptions({ARG_PATH})),
1204 path_("") {
1205 std::map<std::string, std::string>::const_iterator itr =
1206 options.find(ARG_PATH);
1207 if (itr != options.end()) {
1208 path_ = itr->second;
1209 if (path_.empty()) {
1210 exec_state_ = LDBCommandExecuteResult::Failed("--path: missing pathname");
1211 }
1212 }
1213 }
1214
DoCommand()1215 void FileChecksumDumpCommand::DoCommand() {
1216 // print out the checksum information in the following format:
1217 // sst file number, checksum function name, checksum value
1218 // sst file number, checksum function name, checksum value
1219 // ......
1220
1221 std::unique_ptr<FileChecksumList> checksum_list(NewFileChecksumList());
1222 GetLiveFilesChecksumInfoFromVersionSet(options_, db_path_,
1223 checksum_list.get());
1224 if (checksum_list != nullptr) {
1225 std::vector<uint64_t> file_numbers;
1226 std::vector<std::string> checksums;
1227 std::vector<std::string> checksum_func_names;
1228 Status s = checksum_list->GetAllFileChecksums(&file_numbers, &checksums,
1229 &checksum_func_names);
1230 if (s.ok()) {
1231 for (size_t i = 0; i < file_numbers.size(); i++) {
1232 assert(i < file_numbers.size());
1233 assert(i < checksums.size());
1234 assert(i < checksum_func_names.size());
1235 fprintf(stdout, "%" PRId64 ", %s, %s\n", file_numbers[i],
1236 checksum_func_names[i].c_str(), checksums[i].c_str());
1237 }
1238 }
1239 fprintf(stdout, "Print SST file checksum information finished \n");
1240 }
1241 }
1242
1243 // ----------------------------------------------------------------------------
1244
Help(std::string & ret)1245 void ListColumnFamiliesCommand::Help(std::string& ret) {
1246 ret.append(" ");
1247 ret.append(ListColumnFamiliesCommand::Name());
1248 ret.append("\n");
1249 }
1250
ListColumnFamiliesCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)1251 ListColumnFamiliesCommand::ListColumnFamiliesCommand(
1252 const std::vector<std::string>& /*params*/,
1253 const std::map<std::string, std::string>& options,
1254 const std::vector<std::string>& flags)
1255 : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {}
1256
DoCommand()1257 void ListColumnFamiliesCommand::DoCommand() {
1258 std::vector<std::string> column_families;
1259 Status s = DB::ListColumnFamilies(options_, db_path_, &column_families);
1260 if (!s.ok()) {
1261 fprintf(stderr, "Error in processing db %s %s\n", db_path_.c_str(),
1262 s.ToString().c_str());
1263 } else {
1264 fprintf(stdout, "Column families in %s: \n{", db_path_.c_str());
1265 bool first = true;
1266 for (auto cf : column_families) {
1267 if (!first) {
1268 fprintf(stdout, ", ");
1269 }
1270 first = false;
1271 fprintf(stdout, "%s", cf.c_str());
1272 }
1273 fprintf(stdout, "}\n");
1274 }
1275 }
1276
Help(std::string & ret)1277 void CreateColumnFamilyCommand::Help(std::string& ret) {
1278 ret.append(" ");
1279 ret.append(CreateColumnFamilyCommand::Name());
1280 ret.append(" --db=<db_path> <new_column_family_name>");
1281 ret.append("\n");
1282 }
1283
CreateColumnFamilyCommand(const std::vector<std::string> & params,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)1284 CreateColumnFamilyCommand::CreateColumnFamilyCommand(
1285 const std::vector<std::string>& params,
1286 const std::map<std::string, std::string>& options,
1287 const std::vector<std::string>& flags)
1288 : LDBCommand(options, flags, true, {ARG_DB}) {
1289 if (params.size() != 1) {
1290 exec_state_ = LDBCommandExecuteResult::Failed(
1291 "new column family name must be specified");
1292 } else {
1293 new_cf_name_ = params[0];
1294 }
1295 }
1296
DoCommand()1297 void CreateColumnFamilyCommand::DoCommand() {
1298 ColumnFamilyHandle* new_cf_handle = nullptr;
1299 Status st = db_->CreateColumnFamily(options_, new_cf_name_, &new_cf_handle);
1300 if (st.ok()) {
1301 fprintf(stdout, "OK\n");
1302 } else {
1303 exec_state_ = LDBCommandExecuteResult::Failed(
1304 "Fail to create new column family: " + st.ToString());
1305 }
1306 delete new_cf_handle;
1307 CloseDB();
1308 }
1309
Help(std::string & ret)1310 void DropColumnFamilyCommand::Help(std::string& ret) {
1311 ret.append(" ");
1312 ret.append(DropColumnFamilyCommand::Name());
1313 ret.append(" --db=<db_path> <column_family_name_to_drop>");
1314 ret.append("\n");
1315 }
1316
DropColumnFamilyCommand(const std::vector<std::string> & params,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)1317 DropColumnFamilyCommand::DropColumnFamilyCommand(
1318 const std::vector<std::string>& params,
1319 const std::map<std::string, std::string>& options,
1320 const std::vector<std::string>& flags)
1321 : LDBCommand(options, flags, true, {ARG_DB}) {
1322 if (params.size() != 1) {
1323 exec_state_ = LDBCommandExecuteResult::Failed(
1324 "The name of column family to drop must be specified");
1325 } else {
1326 cf_name_to_drop_ = params[0];
1327 }
1328 }
1329
DoCommand()1330 void DropColumnFamilyCommand::DoCommand() {
1331 auto iter = cf_handles_.find(cf_name_to_drop_);
1332 if (iter == cf_handles_.end()) {
1333 exec_state_ = LDBCommandExecuteResult::Failed(
1334 "Column family: " + cf_name_to_drop_ + " doesn't exist in db.");
1335 return;
1336 }
1337 ColumnFamilyHandle* cf_handle_to_drop = iter->second;
1338 Status st = db_->DropColumnFamily(cf_handle_to_drop);
1339 if (st.ok()) {
1340 fprintf(stdout, "OK\n");
1341 } else {
1342 exec_state_ = LDBCommandExecuteResult::Failed(
1343 "Fail to drop column family: " + st.ToString());
1344 }
1345 CloseDB();
1346 }
1347
1348 // ----------------------------------------------------------------------------
1349 namespace {
1350
1351 // This function only called when it's the sane case of >1 buckets in time-range
1352 // Also called only when timekv falls between ttl_start and ttl_end provided
IncBucketCounts(std::vector<uint64_t> & bucket_counts,int ttl_start,int time_range,int bucket_size,int timekv,int num_buckets)1353 void IncBucketCounts(std::vector<uint64_t>& bucket_counts, int ttl_start,
1354 int time_range, int bucket_size, int timekv,
1355 int num_buckets) {
1356 #ifdef NDEBUG
1357 (void)time_range;
1358 (void)num_buckets;
1359 #endif
1360 assert(time_range > 0 && timekv >= ttl_start && bucket_size > 0 &&
1361 timekv < (ttl_start + time_range) && num_buckets > 1);
1362 int bucket = (timekv - ttl_start) / bucket_size;
1363 bucket_counts[bucket]++;
1364 }
1365
PrintBucketCounts(const std::vector<uint64_t> & bucket_counts,int ttl_start,int ttl_end,int bucket_size,int num_buckets)1366 void PrintBucketCounts(const std::vector<uint64_t>& bucket_counts,
1367 int ttl_start, int ttl_end, int bucket_size,
1368 int num_buckets) {
1369 int time_point = ttl_start;
1370 for(int i = 0; i < num_buckets - 1; i++, time_point += bucket_size) {
1371 fprintf(stdout, "Keys in range %s to %s : %lu\n",
1372 TimeToHumanString(time_point).c_str(),
1373 TimeToHumanString(time_point + bucket_size).c_str(),
1374 (unsigned long)bucket_counts[i]);
1375 }
1376 fprintf(stdout, "Keys in range %s to %s : %lu\n",
1377 TimeToHumanString(time_point).c_str(),
1378 TimeToHumanString(ttl_end).c_str(),
1379 (unsigned long)bucket_counts[num_buckets - 1]);
1380 }
1381
1382 } // namespace
1383
1384 const std::string InternalDumpCommand::ARG_COUNT_ONLY = "count_only";
1385 const std::string InternalDumpCommand::ARG_COUNT_DELIM = "count_delim";
1386 const std::string InternalDumpCommand::ARG_STATS = "stats";
1387 const std::string InternalDumpCommand::ARG_INPUT_KEY_HEX = "input_key_hex";
1388
InternalDumpCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)1389 InternalDumpCommand::InternalDumpCommand(
1390 const std::vector<std::string>& /*params*/,
1391 const std::map<std::string, std::string>& options,
1392 const std::vector<std::string>& flags)
1393 : LDBCommand(
1394 options, flags, true,
1395 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
1396 ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY,
1397 ARG_COUNT_DELIM, ARG_STATS, ARG_INPUT_KEY_HEX})),
1398 has_from_(false),
1399 has_to_(false),
1400 max_keys_(-1),
1401 delim_("."),
1402 count_only_(false),
1403 count_delim_(false),
1404 print_stats_(false),
1405 is_input_key_hex_(false) {
1406 has_from_ = ParseStringOption(options, ARG_FROM, &from_);
1407 has_to_ = ParseStringOption(options, ARG_TO, &to_);
1408
1409 ParseIntOption(options, ARG_MAX_KEYS, max_keys_, exec_state_);
1410 std::map<std::string, std::string>::const_iterator itr =
1411 options.find(ARG_COUNT_DELIM);
1412 if (itr != options.end()) {
1413 delim_ = itr->second;
1414 count_delim_ = true;
1415 // fprintf(stdout,"delim = %c\n",delim_[0]);
1416 } else {
1417 count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM);
1418 delim_=".";
1419 }
1420
1421 print_stats_ = IsFlagPresent(flags, ARG_STATS);
1422 count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
1423 is_input_key_hex_ = IsFlagPresent(flags, ARG_INPUT_KEY_HEX);
1424
1425 if (is_input_key_hex_) {
1426 if (has_from_) {
1427 from_ = HexToString(from_);
1428 }
1429 if (has_to_) {
1430 to_ = HexToString(to_);
1431 }
1432 }
1433 }
1434
Help(std::string & ret)1435 void InternalDumpCommand::Help(std::string& ret) {
1436 ret.append(" ");
1437 ret.append(InternalDumpCommand::Name());
1438 ret.append(HelpRangeCmdArgs());
1439 ret.append(" [--" + ARG_INPUT_KEY_HEX + "]");
1440 ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
1441 ret.append(" [--" + ARG_COUNT_ONLY + "]");
1442 ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]");
1443 ret.append(" [--" + ARG_STATS + "]");
1444 ret.append("\n");
1445 }
1446
DoCommand()1447 void InternalDumpCommand::DoCommand() {
1448 if (!db_) {
1449 assert(GetExecuteState().IsFailed());
1450 return;
1451 }
1452
1453 if (print_stats_) {
1454 std::string stats;
1455 if (db_->GetProperty(GetCfHandle(), "rocksdb.stats", &stats)) {
1456 fprintf(stdout, "%s\n", stats.c_str());
1457 }
1458 }
1459
1460 // Cast as DBImpl to get internal iterator
1461 std::vector<KeyVersion> key_versions;
1462 Status st = GetAllKeyVersions(db_, GetCfHandle(), from_, to_, max_keys_,
1463 &key_versions);
1464 if (!st.ok()) {
1465 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
1466 return;
1467 }
1468 std::string rtype1, rtype2, row, val;
1469 rtype2 = "";
1470 uint64_t c=0;
1471 uint64_t s1=0,s2=0;
1472
1473 long long count = 0;
1474 for (auto& key_version : key_versions) {
1475 InternalKey ikey(key_version.user_key, key_version.sequence,
1476 static_cast<ValueType>(key_version.type));
1477 if (has_to_ && ikey.user_key() == to_) {
1478 // GetAllKeyVersions() includes keys with user key `to_`, but idump has
1479 // traditionally excluded such keys.
1480 break;
1481 }
1482 ++count;
1483 int k;
1484 if (count_delim_) {
1485 rtype1 = "";
1486 s1=0;
1487 row = ikey.Encode().ToString();
1488 val = key_version.value;
1489 for(k=0;row[k]!='\x01' && row[k]!='\0';k++)
1490 s1++;
1491 for(k=0;val[k]!='\x01' && val[k]!='\0';k++)
1492 s1++;
1493 for(int j=0;row[j]!=delim_[0] && row[j]!='\0' && row[j]!='\x01';j++)
1494 rtype1+=row[j];
1495 if(rtype2.compare("") && rtype2.compare(rtype1)!=0) {
1496 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
1497 rtype2.c_str(), c, s2);
1498 c=1;
1499 s2=s1;
1500 rtype2 = rtype1;
1501 } else {
1502 c++;
1503 s2+=s1;
1504 rtype2=rtype1;
1505 }
1506 }
1507
1508 if (!count_only_ && !count_delim_) {
1509 std::string key = ikey.DebugString(is_key_hex_);
1510 std::string value = Slice(key_version.value).ToString(is_value_hex_);
1511 std::cout << key << " => " << value << "\n";
1512 }
1513
1514 // Terminate if maximum number of keys have been dumped
1515 if (max_keys_ > 0 && count >= max_keys_) break;
1516 }
1517 if(count_delim_) {
1518 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
1519 rtype2.c_str(), c, s2);
1520 } else {
1521 fprintf(stdout, "Internal keys in range: %lld\n", count);
1522 }
1523 }
1524
1525 const std::string DBDumperCommand::ARG_COUNT_ONLY = "count_only";
1526 const std::string DBDumperCommand::ARG_COUNT_DELIM = "count_delim";
1527 const std::string DBDumperCommand::ARG_STATS = "stats";
1528 const std::string DBDumperCommand::ARG_TTL_BUCKET = "bucket";
1529
DBDumperCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)1530 DBDumperCommand::DBDumperCommand(
1531 const std::vector<std::string>& /*params*/,
1532 const std::map<std::string, std::string>& options,
1533 const std::vector<std::string>& flags)
1534 : LDBCommand(options, flags, true,
1535 BuildCmdLineOptions(
1536 {ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
1537 ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_COUNT_DELIM,
1538 ARG_STATS, ARG_TTL_START, ARG_TTL_END, ARG_TTL_BUCKET,
1539 ARG_TIMESTAMP, ARG_PATH})),
1540 null_from_(true),
1541 null_to_(true),
1542 max_keys_(-1),
1543 count_only_(false),
1544 count_delim_(false),
1545 print_stats_(false) {
1546 std::map<std::string, std::string>::const_iterator itr =
1547 options.find(ARG_FROM);
1548 if (itr != options.end()) {
1549 null_from_ = false;
1550 from_ = itr->second;
1551 }
1552
1553 itr = options.find(ARG_TO);
1554 if (itr != options.end()) {
1555 null_to_ = false;
1556 to_ = itr->second;
1557 }
1558
1559 itr = options.find(ARG_MAX_KEYS);
1560 if (itr != options.end()) {
1561 try {
1562 #if defined(CYGWIN)
1563 max_keys_ = strtol(itr->second.c_str(), 0, 10);
1564 #else
1565 max_keys_ = std::stoi(itr->second);
1566 #endif
1567 } catch (const std::invalid_argument&) {
1568 exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
1569 " has an invalid value");
1570 } catch (const std::out_of_range&) {
1571 exec_state_ = LDBCommandExecuteResult::Failed(
1572 ARG_MAX_KEYS + " has a value out-of-range");
1573 }
1574 }
1575 itr = options.find(ARG_COUNT_DELIM);
1576 if (itr != options.end()) {
1577 delim_ = itr->second;
1578 count_delim_ = true;
1579 } else {
1580 count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM);
1581 delim_=".";
1582 }
1583
1584 print_stats_ = IsFlagPresent(flags, ARG_STATS);
1585 count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
1586
1587 if (is_key_hex_) {
1588 if (!null_from_) {
1589 from_ = HexToString(from_);
1590 }
1591 if (!null_to_) {
1592 to_ = HexToString(to_);
1593 }
1594 }
1595
1596 itr = options.find(ARG_PATH);
1597 if (itr != options.end()) {
1598 path_ = itr->second;
1599 if (db_path_.empty()) {
1600 db_path_ = path_;
1601 }
1602 }
1603 }
1604
Help(std::string & ret)1605 void DBDumperCommand::Help(std::string& ret) {
1606 ret.append(" ");
1607 ret.append(DBDumperCommand::Name());
1608 ret.append(HelpRangeCmdArgs());
1609 ret.append(" [--" + ARG_TTL + "]");
1610 ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
1611 ret.append(" [--" + ARG_TIMESTAMP + "]");
1612 ret.append(" [--" + ARG_COUNT_ONLY + "]");
1613 ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]");
1614 ret.append(" [--" + ARG_STATS + "]");
1615 ret.append(" [--" + ARG_TTL_BUCKET + "=<N>]");
1616 ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
1617 ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
1618 ret.append(" [--" + ARG_PATH + "=<path_to_a_file>]");
1619 ret.append("\n");
1620 }
1621
1622 /**
1623 * Handles two separate cases:
1624 *
1625 * 1) --db is specified - just dump the database.
1626 *
1627 * 2) --path is specified - determine based on file extension what dumping
1628 * function to call. Please note that we intentionally use the extension
1629 * and avoid probing the file contents under the assumption that renaming
1630 * the files is not a supported scenario.
1631 *
1632 */
DoCommand()1633 void DBDumperCommand::DoCommand() {
1634 if (!db_) {
1635 assert(!path_.empty());
1636 std::string fileName = GetFileNameFromPath(path_);
1637 uint64_t number;
1638 FileType type;
1639
1640 exec_state_ = LDBCommandExecuteResult::Succeed("");
1641
1642 if (!ParseFileName(fileName, &number, &type)) {
1643 exec_state_ =
1644 LDBCommandExecuteResult::Failed("Can't parse file type: " + path_);
1645 return;
1646 }
1647
1648 switch (type) {
1649 case kLogFile:
1650 // TODO(myabandeh): allow configuring is_write_commited
1651 DumpWalFile(options_, path_, /* print_header_ */ true,
1652 /* print_values_ */ true, true /* is_write_commited */,
1653 &exec_state_);
1654 break;
1655 case kTableFile:
1656 DumpSstFile(options_, path_, is_key_hex_, /* show_properties */ true);
1657 break;
1658 case kDescriptorFile:
1659 DumpManifestFile(options_, path_, /* verbose_ */ false, is_key_hex_,
1660 /* json_ */ false);
1661 break;
1662 default:
1663 exec_state_ = LDBCommandExecuteResult::Failed(
1664 "File type not supported: " + path_);
1665 break;
1666 }
1667
1668 } else {
1669 DoDumpCommand();
1670 }
1671 }
1672
DoDumpCommand()1673 void DBDumperCommand::DoDumpCommand() {
1674 assert(nullptr != db_);
1675 assert(path_.empty());
1676
1677 // Parse command line args
1678 uint64_t count = 0;
1679 if (print_stats_) {
1680 std::string stats;
1681 if (db_->GetProperty("rocksdb.stats", &stats)) {
1682 fprintf(stdout, "%s\n", stats.c_str());
1683 }
1684 }
1685
1686 // Setup key iterator
1687 ReadOptions scan_read_opts;
1688 scan_read_opts.total_order_seek = true;
1689 Iterator* iter = db_->NewIterator(scan_read_opts, GetCfHandle());
1690 Status st = iter->status();
1691 if (!st.ok()) {
1692 exec_state_ =
1693 LDBCommandExecuteResult::Failed("Iterator error." + st.ToString());
1694 }
1695
1696 if (!null_from_) {
1697 iter->Seek(from_);
1698 } else {
1699 iter->SeekToFirst();
1700 }
1701
1702 int max_keys = max_keys_;
1703 int ttl_start;
1704 if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
1705 ttl_start = DBWithTTLImpl::kMinTimestamp; // TTL introduction time
1706 }
1707 int ttl_end;
1708 if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
1709 ttl_end = DBWithTTLImpl::kMaxTimestamp; // Max time allowed by TTL feature
1710 }
1711 if (ttl_end < ttl_start) {
1712 fprintf(stderr, "Error: End time can't be less than start time\n");
1713 delete iter;
1714 return;
1715 }
1716 int time_range = ttl_end - ttl_start;
1717 int bucket_size;
1718 if (!ParseIntOption(option_map_, ARG_TTL_BUCKET, bucket_size, exec_state_) ||
1719 bucket_size <= 0) {
1720 bucket_size = time_range; // Will have just 1 bucket by default
1721 }
1722 //cretaing variables for row count of each type
1723 std::string rtype1, rtype2, row, val;
1724 rtype2 = "";
1725 uint64_t c=0;
1726 uint64_t s1=0,s2=0;
1727
1728 // At this point, bucket_size=0 => time_range=0
1729 int num_buckets = (bucket_size >= time_range)
1730 ? 1
1731 : ((time_range + bucket_size - 1) / bucket_size);
1732 std::vector<uint64_t> bucket_counts(num_buckets, 0);
1733 if (is_db_ttl_ && !count_only_ && timestamp_ && !count_delim_) {
1734 fprintf(stdout, "Dumping key-values from %s to %s\n",
1735 TimeToHumanString(ttl_start).c_str(),
1736 TimeToHumanString(ttl_end).c_str());
1737 }
1738
1739 HistogramImpl vsize_hist;
1740
1741 for (; iter->Valid(); iter->Next()) {
1742 int rawtime = 0;
1743 // If end marker was specified, we stop before it
1744 if (!null_to_ && (iter->key().ToString() >= to_))
1745 break;
1746 // Terminate if maximum number of keys have been dumped
1747 if (max_keys == 0)
1748 break;
1749 if (is_db_ttl_) {
1750 TtlIterator* it_ttl = static_cast_with_check<TtlIterator, Iterator>(iter);
1751 rawtime = it_ttl->ttl_timestamp();
1752 if (rawtime < ttl_start || rawtime >= ttl_end) {
1753 continue;
1754 }
1755 }
1756 if (max_keys > 0) {
1757 --max_keys;
1758 }
1759 if (is_db_ttl_ && num_buckets > 1) {
1760 IncBucketCounts(bucket_counts, ttl_start, time_range, bucket_size,
1761 rawtime, num_buckets);
1762 }
1763 ++count;
1764 if (count_delim_) {
1765 rtype1 = "";
1766 row = iter->key().ToString();
1767 val = iter->value().ToString();
1768 s1 = row.size()+val.size();
1769 for(int j=0;row[j]!=delim_[0] && row[j]!='\0';j++)
1770 rtype1+=row[j];
1771 if(rtype2.compare("") && rtype2.compare(rtype1)!=0) {
1772 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
1773 rtype2.c_str(), c, s2);
1774 c=1;
1775 s2=s1;
1776 rtype2 = rtype1;
1777 } else {
1778 c++;
1779 s2+=s1;
1780 rtype2=rtype1;
1781 }
1782
1783 }
1784
1785 if (count_only_) {
1786 vsize_hist.Add(iter->value().size());
1787 }
1788
1789 if (!count_only_ && !count_delim_) {
1790 if (is_db_ttl_ && timestamp_) {
1791 fprintf(stdout, "%s ", TimeToHumanString(rawtime).c_str());
1792 }
1793 std::string str =
1794 PrintKeyValue(iter->key().ToString(), iter->value().ToString(),
1795 is_key_hex_, is_value_hex_);
1796 fprintf(stdout, "%s\n", str.c_str());
1797 }
1798 }
1799
1800 if (num_buckets > 1 && is_db_ttl_) {
1801 PrintBucketCounts(bucket_counts, ttl_start, ttl_end, bucket_size,
1802 num_buckets);
1803 } else if(count_delim_) {
1804 fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n",
1805 rtype2.c_str(), c, s2);
1806 } else {
1807 fprintf(stdout, "Keys in range: %" PRIu64 "\n", count);
1808 }
1809
1810 if (count_only_) {
1811 fprintf(stdout, "Value size distribution: \n");
1812 fprintf(stdout, "%s\n", vsize_hist.ToString().c_str());
1813 }
1814 // Clean up
1815 delete iter;
1816 }
1817
1818 const std::string ReduceDBLevelsCommand::ARG_NEW_LEVELS = "new_levels";
1819 const std::string ReduceDBLevelsCommand::ARG_PRINT_OLD_LEVELS =
1820 "print_old_levels";
1821
ReduceDBLevelsCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)1822 ReduceDBLevelsCommand::ReduceDBLevelsCommand(
1823 const std::vector<std::string>& /*params*/,
1824 const std::map<std::string, std::string>& options,
1825 const std::vector<std::string>& flags)
1826 : LDBCommand(options, flags, false,
1827 BuildCmdLineOptions({ARG_NEW_LEVELS, ARG_PRINT_OLD_LEVELS})),
1828 old_levels_(1 << 7),
1829 new_levels_(-1),
1830 print_old_levels_(false) {
1831 ParseIntOption(option_map_, ARG_NEW_LEVELS, new_levels_, exec_state_);
1832 print_old_levels_ = IsFlagPresent(flags, ARG_PRINT_OLD_LEVELS);
1833
1834 if(new_levels_ <= 0) {
1835 exec_state_ = LDBCommandExecuteResult::Failed(
1836 " Use --" + ARG_NEW_LEVELS + " to specify a new level number\n");
1837 }
1838 }
1839
PrepareArgs(const std::string & db_path,int new_levels,bool print_old_level)1840 std::vector<std::string> ReduceDBLevelsCommand::PrepareArgs(
1841 const std::string& db_path, int new_levels, bool print_old_level) {
1842 std::vector<std::string> ret;
1843 ret.push_back("reduce_levels");
1844 ret.push_back("--" + ARG_DB + "=" + db_path);
1845 ret.push_back("--" + ARG_NEW_LEVELS + "=" +
1846 ROCKSDB_NAMESPACE::ToString(new_levels));
1847 if(print_old_level) {
1848 ret.push_back("--" + ARG_PRINT_OLD_LEVELS);
1849 }
1850 return ret;
1851 }
1852
Help(std::string & ret)1853 void ReduceDBLevelsCommand::Help(std::string& ret) {
1854 ret.append(" ");
1855 ret.append(ReduceDBLevelsCommand::Name());
1856 ret.append(" --" + ARG_NEW_LEVELS + "=<New number of levels>");
1857 ret.append(" [--" + ARG_PRINT_OLD_LEVELS + "]");
1858 ret.append("\n");
1859 }
1860
PrepareOptionsForOpenDB()1861 Options ReduceDBLevelsCommand::PrepareOptionsForOpenDB() {
1862 Options opt = LDBCommand::PrepareOptionsForOpenDB();
1863 opt.num_levels = old_levels_;
1864 opt.max_bytes_for_level_multiplier_additional.resize(opt.num_levels, 1);
1865 // Disable size compaction
1866 opt.max_bytes_for_level_base = 1ULL << 50;
1867 opt.max_bytes_for_level_multiplier = 1;
1868 return opt;
1869 }
1870
GetOldNumOfLevels(Options & opt,int * levels)1871 Status ReduceDBLevelsCommand::GetOldNumOfLevels(Options& opt,
1872 int* levels) {
1873 ImmutableDBOptions db_options(opt);
1874 EnvOptions soptions;
1875 std::shared_ptr<Cache> tc(
1876 NewLRUCache(opt.max_open_files - 10, opt.table_cache_numshardbits));
1877 const InternalKeyComparator cmp(opt.comparator);
1878 WriteController wc(opt.delayed_write_rate);
1879 WriteBufferManager wb(opt.db_write_buffer_size);
1880 VersionSet versions(db_path_, &db_options, soptions, tc.get(), &wb, &wc,
1881 /*block_cache_tracer=*/nullptr);
1882 std::vector<ColumnFamilyDescriptor> dummy;
1883 ColumnFamilyDescriptor dummy_descriptor(kDefaultColumnFamilyName,
1884 ColumnFamilyOptions(opt));
1885 dummy.push_back(dummy_descriptor);
1886 // We rely the VersionSet::Recover to tell us the internal data structures
1887 // in the db. And the Recover() should never do any change
1888 // (like LogAndApply) to the manifest file.
1889 Status st = versions.Recover(dummy);
1890 if (!st.ok()) {
1891 return st;
1892 }
1893 int max = -1;
1894 auto default_cfd = versions.GetColumnFamilySet()->GetDefault();
1895 for (int i = 0; i < default_cfd->NumberLevels(); i++) {
1896 if (default_cfd->current()->storage_info()->NumLevelFiles(i)) {
1897 max = i;
1898 }
1899 }
1900
1901 *levels = max + 1;
1902 return st;
1903 }
1904
DoCommand()1905 void ReduceDBLevelsCommand::DoCommand() {
1906 if (new_levels_ <= 1) {
1907 exec_state_ =
1908 LDBCommandExecuteResult::Failed("Invalid number of levels.\n");
1909 return;
1910 }
1911
1912 Status st;
1913 Options opt = PrepareOptionsForOpenDB();
1914 int old_level_num = -1;
1915 st = GetOldNumOfLevels(opt, &old_level_num);
1916 if (!st.ok()) {
1917 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
1918 return;
1919 }
1920
1921 if (print_old_levels_) {
1922 fprintf(stdout, "The old number of levels in use is %d\n", old_level_num);
1923 }
1924
1925 if (old_level_num <= new_levels_) {
1926 return;
1927 }
1928
1929 old_levels_ = old_level_num;
1930
1931 OpenDB();
1932 if (exec_state_.IsFailed()) {
1933 return;
1934 }
1935 assert(db_ != nullptr);
1936 // Compact the whole DB to put all files to the highest level.
1937 fprintf(stdout, "Compacting the db...\n");
1938 db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr);
1939 CloseDB();
1940
1941 EnvOptions soptions;
1942 st = VersionSet::ReduceNumberOfLevels(db_path_, &opt, soptions, new_levels_);
1943 if (!st.ok()) {
1944 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
1945 return;
1946 }
1947 }
1948
1949 const std::string ChangeCompactionStyleCommand::ARG_OLD_COMPACTION_STYLE =
1950 "old_compaction_style";
1951 const std::string ChangeCompactionStyleCommand::ARG_NEW_COMPACTION_STYLE =
1952 "new_compaction_style";
1953
ChangeCompactionStyleCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)1954 ChangeCompactionStyleCommand::ChangeCompactionStyleCommand(
1955 const std::vector<std::string>& /*params*/,
1956 const std::map<std::string, std::string>& options,
1957 const std::vector<std::string>& flags)
1958 : LDBCommand(options, flags, false,
1959 BuildCmdLineOptions(
1960 {ARG_OLD_COMPACTION_STYLE, ARG_NEW_COMPACTION_STYLE})),
1961 old_compaction_style_(-1),
1962 new_compaction_style_(-1) {
1963 ParseIntOption(option_map_, ARG_OLD_COMPACTION_STYLE, old_compaction_style_,
1964 exec_state_);
1965 if (old_compaction_style_ != kCompactionStyleLevel &&
1966 old_compaction_style_ != kCompactionStyleUniversal) {
1967 exec_state_ = LDBCommandExecuteResult::Failed(
1968 "Use --" + ARG_OLD_COMPACTION_STYLE + " to specify old compaction " +
1969 "style. Check ldb help for proper compaction style value.\n");
1970 return;
1971 }
1972
1973 ParseIntOption(option_map_, ARG_NEW_COMPACTION_STYLE, new_compaction_style_,
1974 exec_state_);
1975 if (new_compaction_style_ != kCompactionStyleLevel &&
1976 new_compaction_style_ != kCompactionStyleUniversal) {
1977 exec_state_ = LDBCommandExecuteResult::Failed(
1978 "Use --" + ARG_NEW_COMPACTION_STYLE + " to specify new compaction " +
1979 "style. Check ldb help for proper compaction style value.\n");
1980 return;
1981 }
1982
1983 if (new_compaction_style_ == old_compaction_style_) {
1984 exec_state_ = LDBCommandExecuteResult::Failed(
1985 "Old compaction style is the same as new compaction style. "
1986 "Nothing to do.\n");
1987 return;
1988 }
1989
1990 if (old_compaction_style_ == kCompactionStyleUniversal &&
1991 new_compaction_style_ == kCompactionStyleLevel) {
1992 exec_state_ = LDBCommandExecuteResult::Failed(
1993 "Convert from universal compaction to level compaction. "
1994 "Nothing to do.\n");
1995 return;
1996 }
1997 }
1998
Help(std::string & ret)1999 void ChangeCompactionStyleCommand::Help(std::string& ret) {
2000 ret.append(" ");
2001 ret.append(ChangeCompactionStyleCommand::Name());
2002 ret.append(" --" + ARG_OLD_COMPACTION_STYLE + "=<Old compaction style: 0 " +
2003 "for level compaction, 1 for universal compaction>");
2004 ret.append(" --" + ARG_NEW_COMPACTION_STYLE + "=<New compaction style: 0 " +
2005 "for level compaction, 1 for universal compaction>");
2006 ret.append("\n");
2007 }
2008
PrepareOptionsForOpenDB()2009 Options ChangeCompactionStyleCommand::PrepareOptionsForOpenDB() {
2010 Options opt = LDBCommand::PrepareOptionsForOpenDB();
2011
2012 if (old_compaction_style_ == kCompactionStyleLevel &&
2013 new_compaction_style_ == kCompactionStyleUniversal) {
2014 // In order to convert from level compaction to universal compaction, we
2015 // need to compact all data into a single file and move it to level 0.
2016 opt.disable_auto_compactions = true;
2017 opt.target_file_size_base = INT_MAX;
2018 opt.target_file_size_multiplier = 1;
2019 opt.max_bytes_for_level_base = INT_MAX;
2020 opt.max_bytes_for_level_multiplier = 1;
2021 }
2022
2023 return opt;
2024 }
2025
DoCommand()2026 void ChangeCompactionStyleCommand::DoCommand() {
2027 // print db stats before we have made any change
2028 std::string property;
2029 std::string files_per_level;
2030 for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) {
2031 db_->GetProperty(GetCfHandle(),
2032 "rocksdb.num-files-at-level" + NumberToString(i),
2033 &property);
2034
2035 // format print string
2036 char buf[100];
2037 snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str());
2038 files_per_level += buf;
2039 }
2040 fprintf(stdout, "files per level before compaction: %s\n",
2041 files_per_level.c_str());
2042
2043 // manual compact into a single file and move the file to level 0
2044 CompactRangeOptions compact_options;
2045 compact_options.change_level = true;
2046 compact_options.target_level = 0;
2047 db_->CompactRange(compact_options, GetCfHandle(), nullptr, nullptr);
2048
2049 // verify compaction result
2050 files_per_level = "";
2051 int num_files = 0;
2052 for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) {
2053 db_->GetProperty(GetCfHandle(),
2054 "rocksdb.num-files-at-level" + NumberToString(i),
2055 &property);
2056
2057 // format print string
2058 char buf[100];
2059 snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str());
2060 files_per_level += buf;
2061
2062 num_files = atoi(property.c_str());
2063
2064 // level 0 should have only 1 file
2065 if (i == 0 && num_files != 1) {
2066 exec_state_ = LDBCommandExecuteResult::Failed(
2067 "Number of db files at "
2068 "level 0 after compaction is " +
2069 ToString(num_files) + ", not 1.\n");
2070 return;
2071 }
2072 // other levels should have no file
2073 if (i > 0 && num_files != 0) {
2074 exec_state_ = LDBCommandExecuteResult::Failed(
2075 "Number of db files at "
2076 "level " +
2077 ToString(i) + " after compaction is " + ToString(num_files) +
2078 ", not 0.\n");
2079 return;
2080 }
2081 }
2082
2083 fprintf(stdout, "files per level after compaction: %s\n",
2084 files_per_level.c_str());
2085 }
2086
2087 // ----------------------------------------------------------------------------
2088
2089 namespace {
2090
2091 struct StdErrReporter : public log::Reader::Reporter {
CorruptionROCKSDB_NAMESPACE::__anon0228c5f50711::StdErrReporter2092 void Corruption(size_t /*bytes*/, const Status& s) override {
2093 std::cerr << "Corruption detected in log file " << s.ToString() << "\n";
2094 }
2095 };
2096
2097 class InMemoryHandler : public WriteBatch::Handler {
2098 public:
InMemoryHandler(std::stringstream & row,bool print_values,bool write_after_commit=false)2099 InMemoryHandler(std::stringstream& row, bool print_values,
2100 bool write_after_commit = false)
2101 : Handler(),
2102 row_(row),
2103 print_values_(print_values),
2104 write_after_commit_(write_after_commit) {}
2105
commonPutMerge(const Slice & key,const Slice & value)2106 void commonPutMerge(const Slice& key, const Slice& value) {
2107 std::string k = LDBCommand::StringToHex(key.ToString());
2108 if (print_values_) {
2109 std::string v = LDBCommand::StringToHex(value.ToString());
2110 row_ << k << " : ";
2111 row_ << v << " ";
2112 } else {
2113 row_ << k << " ";
2114 }
2115 }
2116
PutCF(uint32_t cf,const Slice & key,const Slice & value)2117 Status PutCF(uint32_t cf, const Slice& key, const Slice& value) override {
2118 row_ << "PUT(" << cf << ") : ";
2119 commonPutMerge(key, value);
2120 return Status::OK();
2121 }
2122
MergeCF(uint32_t cf,const Slice & key,const Slice & value)2123 Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override {
2124 row_ << "MERGE(" << cf << ") : ";
2125 commonPutMerge(key, value);
2126 return Status::OK();
2127 }
2128
MarkNoop(bool)2129 Status MarkNoop(bool) override {
2130 row_ << "NOOP ";
2131 return Status::OK();
2132 }
2133
DeleteCF(uint32_t cf,const Slice & key)2134 Status DeleteCF(uint32_t cf, const Slice& key) override {
2135 row_ << "DELETE(" << cf << ") : ";
2136 row_ << LDBCommand::StringToHex(key.ToString()) << " ";
2137 return Status::OK();
2138 }
2139
SingleDeleteCF(uint32_t cf,const Slice & key)2140 Status SingleDeleteCF(uint32_t cf, const Slice& key) override {
2141 row_ << "SINGLE_DELETE(" << cf << ") : ";
2142 row_ << LDBCommand::StringToHex(key.ToString()) << " ";
2143 return Status::OK();
2144 }
2145
DeleteRangeCF(uint32_t cf,const Slice & begin_key,const Slice & end_key)2146 Status DeleteRangeCF(uint32_t cf, const Slice& begin_key,
2147 const Slice& end_key) override {
2148 row_ << "DELETE_RANGE(" << cf << ") : ";
2149 row_ << LDBCommand::StringToHex(begin_key.ToString()) << " ";
2150 row_ << LDBCommand::StringToHex(end_key.ToString()) << " ";
2151 return Status::OK();
2152 }
2153
MarkBeginPrepare(bool unprepare)2154 Status MarkBeginPrepare(bool unprepare) override {
2155 row_ << "BEGIN_PREPARE(";
2156 row_ << (unprepare ? "true" : "false") << ") ";
2157 return Status::OK();
2158 }
2159
MarkEndPrepare(const Slice & xid)2160 Status MarkEndPrepare(const Slice& xid) override {
2161 row_ << "END_PREPARE(";
2162 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
2163 return Status::OK();
2164 }
2165
MarkRollback(const Slice & xid)2166 Status MarkRollback(const Slice& xid) override {
2167 row_ << "ROLLBACK(";
2168 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
2169 return Status::OK();
2170 }
2171
MarkCommit(const Slice & xid)2172 Status MarkCommit(const Slice& xid) override {
2173 row_ << "COMMIT(";
2174 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
2175 return Status::OK();
2176 }
2177
~InMemoryHandler()2178 ~InMemoryHandler() override {}
2179
2180 protected:
WriteAfterCommit() const2181 bool WriteAfterCommit() const override { return write_after_commit_; }
2182
2183 private:
2184 std::stringstream& row_;
2185 bool print_values_;
2186 bool write_after_commit_;
2187 };
2188
DumpWalFile(Options options,std::string wal_file,bool print_header,bool print_values,bool is_write_committed,LDBCommandExecuteResult * exec_state)2189 void DumpWalFile(Options options, std::string wal_file, bool print_header,
2190 bool print_values, bool is_write_committed,
2191 LDBCommandExecuteResult* exec_state) {
2192 Env* env = options.env;
2193 EnvOptions soptions(options);
2194 std::unique_ptr<SequentialFileReader> wal_file_reader;
2195
2196 Status status;
2197 {
2198 std::unique_ptr<SequentialFile> file;
2199 status = env->NewSequentialFile(wal_file, &file, soptions);
2200 if (status.ok()) {
2201 wal_file_reader.reset(new SequentialFileReader(
2202 NewLegacySequentialFileWrapper(file), wal_file));
2203 }
2204 }
2205 if (!status.ok()) {
2206 if (exec_state) {
2207 *exec_state = LDBCommandExecuteResult::Failed("Failed to open WAL file " +
2208 status.ToString());
2209 } else {
2210 std::cerr << "Error: Failed to open WAL file " << status.ToString()
2211 << std::endl;
2212 }
2213 } else {
2214 StdErrReporter reporter;
2215 uint64_t log_number;
2216 FileType type;
2217
2218 // we need the log number, but ParseFilename expects dbname/NNN.log.
2219 std::string sanitized = wal_file;
2220 size_t lastslash = sanitized.rfind('/');
2221 if (lastslash != std::string::npos)
2222 sanitized = sanitized.substr(lastslash + 1);
2223 if (!ParseFileName(sanitized, &log_number, &type)) {
2224 // bogus input, carry on as best we can
2225 log_number = 0;
2226 }
2227 log::Reader reader(options.info_log, std::move(wal_file_reader), &reporter,
2228 true /* checksum */, log_number);
2229 std::string scratch;
2230 WriteBatch batch;
2231 Slice record;
2232 std::stringstream row;
2233 if (print_header) {
2234 std::cout << "Sequence,Count,ByteSize,Physical Offset,Key(s)";
2235 if (print_values) {
2236 std::cout << " : value ";
2237 }
2238 std::cout << "\n";
2239 }
2240 while (reader.ReadRecord(&record, &scratch)) {
2241 row.str("");
2242 if (record.size() < WriteBatchInternal::kHeader) {
2243 reporter.Corruption(record.size(),
2244 Status::Corruption("log record too small"));
2245 } else {
2246 WriteBatchInternal::SetContents(&batch, record);
2247 row << WriteBatchInternal::Sequence(&batch) << ",";
2248 row << WriteBatchInternal::Count(&batch) << ",";
2249 row << WriteBatchInternal::ByteSize(&batch) << ",";
2250 row << reader.LastRecordOffset() << ",";
2251 InMemoryHandler handler(row, print_values, is_write_committed);
2252 batch.Iterate(&handler);
2253 row << "\n";
2254 }
2255 std::cout << row.str();
2256 }
2257 }
2258 }
2259
2260 } // namespace
2261
2262 const std::string WALDumperCommand::ARG_WAL_FILE = "walfile";
2263 const std::string WALDumperCommand::ARG_WRITE_COMMITTED = "write_committed";
2264 const std::string WALDumperCommand::ARG_PRINT_VALUE = "print_value";
2265 const std::string WALDumperCommand::ARG_PRINT_HEADER = "header";
2266
WALDumperCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2267 WALDumperCommand::WALDumperCommand(
2268 const std::vector<std::string>& /*params*/,
2269 const std::map<std::string, std::string>& options,
2270 const std::vector<std::string>& flags)
2271 : LDBCommand(options, flags, true,
2272 BuildCmdLineOptions({ARG_WAL_FILE, ARG_WRITE_COMMITTED,
2273 ARG_PRINT_HEADER, ARG_PRINT_VALUE})),
2274 print_header_(false),
2275 print_values_(false),
2276 is_write_committed_(false) {
2277 wal_file_.clear();
2278
2279 std::map<std::string, std::string>::const_iterator itr =
2280 options.find(ARG_WAL_FILE);
2281 if (itr != options.end()) {
2282 wal_file_ = itr->second;
2283 }
2284
2285
2286 print_header_ = IsFlagPresent(flags, ARG_PRINT_HEADER);
2287 print_values_ = IsFlagPresent(flags, ARG_PRINT_VALUE);
2288 is_write_committed_ = ParseBooleanOption(options, ARG_WRITE_COMMITTED, true);
2289
2290 if (wal_file_.empty()) {
2291 exec_state_ = LDBCommandExecuteResult::Failed("Argument " + ARG_WAL_FILE +
2292 " must be specified.");
2293 }
2294 }
2295
Help(std::string & ret)2296 void WALDumperCommand::Help(std::string& ret) {
2297 ret.append(" ");
2298 ret.append(WALDumperCommand::Name());
2299 ret.append(" --" + ARG_WAL_FILE + "=<write_ahead_log_file_path>");
2300 ret.append(" [--" + ARG_PRINT_HEADER + "] ");
2301 ret.append(" [--" + ARG_PRINT_VALUE + "] ");
2302 ret.append(" [--" + ARG_WRITE_COMMITTED + "=true|false] ");
2303 ret.append("\n");
2304 }
2305
DoCommand()2306 void WALDumperCommand::DoCommand() {
2307 DumpWalFile(options_, wal_file_, print_header_, print_values_,
2308 is_write_committed_, &exec_state_);
2309 }
2310
2311 // ----------------------------------------------------------------------------
2312
GetCommand(const std::vector<std::string> & params,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2313 GetCommand::GetCommand(const std::vector<std::string>& params,
2314 const std::map<std::string, std::string>& options,
2315 const std::vector<std::string>& flags)
2316 : LDBCommand(
2317 options, flags, true,
2318 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2319 if (params.size() != 1) {
2320 exec_state_ = LDBCommandExecuteResult::Failed(
2321 "<key> must be specified for the get command");
2322 } else {
2323 key_ = params.at(0);
2324 }
2325
2326 if (is_key_hex_) {
2327 key_ = HexToString(key_);
2328 }
2329 }
2330
Help(std::string & ret)2331 void GetCommand::Help(std::string& ret) {
2332 ret.append(" ");
2333 ret.append(GetCommand::Name());
2334 ret.append(" <key>");
2335 ret.append(" [--" + ARG_TTL + "]");
2336 ret.append("\n");
2337 }
2338
DoCommand()2339 void GetCommand::DoCommand() {
2340 if (!db_) {
2341 assert(GetExecuteState().IsFailed());
2342 return;
2343 }
2344 std::string value;
2345 Status st = db_->Get(ReadOptions(), GetCfHandle(), key_, &value);
2346 if (st.ok()) {
2347 fprintf(stdout, "%s\n",
2348 (is_value_hex_ ? StringToHex(value) : value).c_str());
2349 } else {
2350 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2351 }
2352 }
2353
2354 // ----------------------------------------------------------------------------
2355
ApproxSizeCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2356 ApproxSizeCommand::ApproxSizeCommand(
2357 const std::vector<std::string>& /*params*/,
2358 const std::map<std::string, std::string>& options,
2359 const std::vector<std::string>& flags)
2360 : LDBCommand(options, flags, true,
2361 BuildCmdLineOptions(
2362 {ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO})) {
2363 if (options.find(ARG_FROM) != options.end()) {
2364 start_key_ = options.find(ARG_FROM)->second;
2365 } else {
2366 exec_state_ = LDBCommandExecuteResult::Failed(
2367 ARG_FROM + " must be specified for approxsize command");
2368 return;
2369 }
2370
2371 if (options.find(ARG_TO) != options.end()) {
2372 end_key_ = options.find(ARG_TO)->second;
2373 } else {
2374 exec_state_ = LDBCommandExecuteResult::Failed(
2375 ARG_TO + " must be specified for approxsize command");
2376 return;
2377 }
2378
2379 if (is_key_hex_) {
2380 start_key_ = HexToString(start_key_);
2381 end_key_ = HexToString(end_key_);
2382 }
2383 }
2384
Help(std::string & ret)2385 void ApproxSizeCommand::Help(std::string& ret) {
2386 ret.append(" ");
2387 ret.append(ApproxSizeCommand::Name());
2388 ret.append(HelpRangeCmdArgs());
2389 ret.append("\n");
2390 }
2391
DoCommand()2392 void ApproxSizeCommand::DoCommand() {
2393 if (!db_) {
2394 assert(GetExecuteState().IsFailed());
2395 return;
2396 }
2397 Range ranges[1];
2398 ranges[0] = Range(start_key_, end_key_);
2399 uint64_t sizes[1];
2400 db_->GetApproximateSizes(GetCfHandle(), ranges, 1, sizes);
2401 fprintf(stdout, "%lu\n", (unsigned long)sizes[0]);
2402 /* Weird that GetApproximateSizes() returns void, although documentation
2403 * says that it returns a Status object.
2404 if (!st.ok()) {
2405 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2406 }
2407 */
2408 }
2409
2410 // ----------------------------------------------------------------------------
2411
BatchPutCommand(const std::vector<std::string> & params,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2412 BatchPutCommand::BatchPutCommand(
2413 const std::vector<std::string>& params,
2414 const std::map<std::string, std::string>& options,
2415 const std::vector<std::string>& flags)
2416 : LDBCommand(options, flags, false,
2417 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
2418 ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) {
2419 if (params.size() < 2) {
2420 exec_state_ = LDBCommandExecuteResult::Failed(
2421 "At least one <key> <value> pair must be specified batchput.");
2422 } else if (params.size() % 2 != 0) {
2423 exec_state_ = LDBCommandExecuteResult::Failed(
2424 "Equal number of <key>s and <value>s must be specified for batchput.");
2425 } else {
2426 for (size_t i = 0; i < params.size(); i += 2) {
2427 std::string key = params.at(i);
2428 std::string value = params.at(i + 1);
2429 key_values_.push_back(std::pair<std::string, std::string>(
2430 is_key_hex_ ? HexToString(key) : key,
2431 is_value_hex_ ? HexToString(value) : value));
2432 }
2433 }
2434 create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
2435 }
2436
Help(std::string & ret)2437 void BatchPutCommand::Help(std::string& ret) {
2438 ret.append(" ");
2439 ret.append(BatchPutCommand::Name());
2440 ret.append(" <key> <value> [<key> <value>] [..]");
2441 ret.append(" [--" + ARG_TTL + "]");
2442 ret.append("\n");
2443 }
2444
DoCommand()2445 void BatchPutCommand::DoCommand() {
2446 if (!db_) {
2447 assert(GetExecuteState().IsFailed());
2448 return;
2449 }
2450 WriteBatch batch;
2451
2452 for (std::vector<std::pair<std::string, std::string>>::const_iterator itr =
2453 key_values_.begin();
2454 itr != key_values_.end(); ++itr) {
2455 batch.Put(GetCfHandle(), itr->first, itr->second);
2456 }
2457 Status st = db_->Write(WriteOptions(), &batch);
2458 if (st.ok()) {
2459 fprintf(stdout, "OK\n");
2460 } else {
2461 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2462 }
2463 }
2464
PrepareOptionsForOpenDB()2465 Options BatchPutCommand::PrepareOptionsForOpenDB() {
2466 Options opt = LDBCommand::PrepareOptionsForOpenDB();
2467 opt.create_if_missing = create_if_missing_;
2468 return opt;
2469 }
2470
2471 // ----------------------------------------------------------------------------
2472
ScanCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2473 ScanCommand::ScanCommand(const std::vector<std::string>& /*params*/,
2474 const std::map<std::string, std::string>& options,
2475 const std::vector<std::string>& flags)
2476 : LDBCommand(
2477 options, flags, true,
2478 BuildCmdLineOptions({ARG_TTL, ARG_NO_VALUE, ARG_HEX, ARG_KEY_HEX,
2479 ARG_TO, ARG_VALUE_HEX, ARG_FROM, ARG_TIMESTAMP,
2480 ARG_MAX_KEYS, ARG_TTL_START, ARG_TTL_END})),
2481 start_key_specified_(false),
2482 end_key_specified_(false),
2483 max_keys_scanned_(-1),
2484 no_value_(false) {
2485 std::map<std::string, std::string>::const_iterator itr =
2486 options.find(ARG_FROM);
2487 if (itr != options.end()) {
2488 start_key_ = itr->second;
2489 if (is_key_hex_) {
2490 start_key_ = HexToString(start_key_);
2491 }
2492 start_key_specified_ = true;
2493 }
2494 itr = options.find(ARG_TO);
2495 if (itr != options.end()) {
2496 end_key_ = itr->second;
2497 if (is_key_hex_) {
2498 end_key_ = HexToString(end_key_);
2499 }
2500 end_key_specified_ = true;
2501 }
2502
2503 std::vector<std::string>::const_iterator vitr =
2504 std::find(flags.begin(), flags.end(), ARG_NO_VALUE);
2505 if (vitr != flags.end()) {
2506 no_value_ = true;
2507 }
2508
2509 itr = options.find(ARG_MAX_KEYS);
2510 if (itr != options.end()) {
2511 try {
2512 #if defined(CYGWIN)
2513 max_keys_scanned_ = strtol(itr->second.c_str(), 0, 10);
2514 #else
2515 max_keys_scanned_ = std::stoi(itr->second);
2516 #endif
2517 } catch (const std::invalid_argument&) {
2518 exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
2519 " has an invalid value");
2520 } catch (const std::out_of_range&) {
2521 exec_state_ = LDBCommandExecuteResult::Failed(
2522 ARG_MAX_KEYS + " has a value out-of-range");
2523 }
2524 }
2525 }
2526
Help(std::string & ret)2527 void ScanCommand::Help(std::string& ret) {
2528 ret.append(" ");
2529 ret.append(ScanCommand::Name());
2530 ret.append(HelpRangeCmdArgs());
2531 ret.append(" [--" + ARG_TTL + "]");
2532 ret.append(" [--" + ARG_TIMESTAMP + "]");
2533 ret.append(" [--" + ARG_MAX_KEYS + "=<N>q] ");
2534 ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
2535 ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
2536 ret.append(" [--" + ARG_NO_VALUE + "]");
2537 ret.append("\n");
2538 }
2539
DoCommand()2540 void ScanCommand::DoCommand() {
2541 if (!db_) {
2542 assert(GetExecuteState().IsFailed());
2543 return;
2544 }
2545
2546 int num_keys_scanned = 0;
2547 ReadOptions scan_read_opts;
2548 scan_read_opts.total_order_seek = true;
2549 Iterator* it = db_->NewIterator(scan_read_opts, GetCfHandle());
2550 if (start_key_specified_) {
2551 it->Seek(start_key_);
2552 } else {
2553 it->SeekToFirst();
2554 }
2555 int ttl_start;
2556 if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
2557 ttl_start = DBWithTTLImpl::kMinTimestamp; // TTL introduction time
2558 }
2559 int ttl_end;
2560 if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
2561 ttl_end = DBWithTTLImpl::kMaxTimestamp; // Max time allowed by TTL feature
2562 }
2563 if (ttl_end < ttl_start) {
2564 fprintf(stderr, "Error: End time can't be less than start time\n");
2565 delete it;
2566 return;
2567 }
2568 if (is_db_ttl_ && timestamp_) {
2569 fprintf(stdout, "Scanning key-values from %s to %s\n",
2570 TimeToHumanString(ttl_start).c_str(),
2571 TimeToHumanString(ttl_end).c_str());
2572 }
2573 for ( ;
2574 it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_);
2575 it->Next()) {
2576 if (is_db_ttl_) {
2577 TtlIterator* it_ttl = static_cast_with_check<TtlIterator, Iterator>(it);
2578 int rawtime = it_ttl->ttl_timestamp();
2579 if (rawtime < ttl_start || rawtime >= ttl_end) {
2580 continue;
2581 }
2582 if (timestamp_) {
2583 fprintf(stdout, "%s ", TimeToHumanString(rawtime).c_str());
2584 }
2585 }
2586
2587 Slice key_slice = it->key();
2588
2589 std::string formatted_key;
2590 if (is_key_hex_) {
2591 formatted_key = "0x" + key_slice.ToString(true /* hex */);
2592 key_slice = formatted_key;
2593 } else if (ldb_options_.key_formatter) {
2594 formatted_key = ldb_options_.key_formatter->Format(key_slice);
2595 key_slice = formatted_key;
2596 }
2597
2598 if (no_value_) {
2599 fprintf(stdout, "%.*s\n", static_cast<int>(key_slice.size()),
2600 key_slice.data());
2601 } else {
2602 Slice val_slice = it->value();
2603 std::string formatted_value;
2604 if (is_value_hex_) {
2605 formatted_value = "0x" + val_slice.ToString(true /* hex */);
2606 val_slice = formatted_value;
2607 }
2608 fprintf(stdout, "%.*s : %.*s\n", static_cast<int>(key_slice.size()),
2609 key_slice.data(), static_cast<int>(val_slice.size()),
2610 val_slice.data());
2611 }
2612
2613 num_keys_scanned++;
2614 if (max_keys_scanned_ >= 0 && num_keys_scanned >= max_keys_scanned_) {
2615 break;
2616 }
2617 }
2618 if (!it->status().ok()) { // Check for any errors found during the scan
2619 exec_state_ = LDBCommandExecuteResult::Failed(it->status().ToString());
2620 }
2621 delete it;
2622 }
2623
2624 // ----------------------------------------------------------------------------
2625
DeleteCommand(const std::vector<std::string> & params,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2626 DeleteCommand::DeleteCommand(const std::vector<std::string>& params,
2627 const std::map<std::string, std::string>& options,
2628 const std::vector<std::string>& flags)
2629 : LDBCommand(options, flags, false,
2630 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2631 if (params.size() != 1) {
2632 exec_state_ = LDBCommandExecuteResult::Failed(
2633 "KEY must be specified for the delete command");
2634 } else {
2635 key_ = params.at(0);
2636 if (is_key_hex_) {
2637 key_ = HexToString(key_);
2638 }
2639 }
2640 }
2641
Help(std::string & ret)2642 void DeleteCommand::Help(std::string& ret) {
2643 ret.append(" ");
2644 ret.append(DeleteCommand::Name() + " <key>");
2645 ret.append("\n");
2646 }
2647
DoCommand()2648 void DeleteCommand::DoCommand() {
2649 if (!db_) {
2650 assert(GetExecuteState().IsFailed());
2651 return;
2652 }
2653 Status st = db_->Delete(WriteOptions(), GetCfHandle(), key_);
2654 if (st.ok()) {
2655 fprintf(stdout, "OK\n");
2656 } else {
2657 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2658 }
2659 }
2660
DeleteRangeCommand(const std::vector<std::string> & params,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2661 DeleteRangeCommand::DeleteRangeCommand(
2662 const std::vector<std::string>& params,
2663 const std::map<std::string, std::string>& options,
2664 const std::vector<std::string>& flags)
2665 : LDBCommand(options, flags, false,
2666 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2667 if (params.size() != 2) {
2668 exec_state_ = LDBCommandExecuteResult::Failed(
2669 "begin and end keys must be specified for the delete command");
2670 } else {
2671 begin_key_ = params.at(0);
2672 end_key_ = params.at(1);
2673 if (is_key_hex_) {
2674 begin_key_ = HexToString(begin_key_);
2675 end_key_ = HexToString(end_key_);
2676 }
2677 }
2678 }
2679
Help(std::string & ret)2680 void DeleteRangeCommand::Help(std::string& ret) {
2681 ret.append(" ");
2682 ret.append(DeleteRangeCommand::Name() + " <begin key> <end key>");
2683 ret.append("\n");
2684 }
2685
DoCommand()2686 void DeleteRangeCommand::DoCommand() {
2687 if (!db_) {
2688 assert(GetExecuteState().IsFailed());
2689 return;
2690 }
2691 Status st =
2692 db_->DeleteRange(WriteOptions(), GetCfHandle(), begin_key_, end_key_);
2693 if (st.ok()) {
2694 fprintf(stdout, "OK\n");
2695 } else {
2696 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2697 }
2698 }
2699
PutCommand(const std::vector<std::string> & params,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2700 PutCommand::PutCommand(const std::vector<std::string>& params,
2701 const std::map<std::string, std::string>& options,
2702 const std::vector<std::string>& flags)
2703 : LDBCommand(options, flags, false,
2704 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
2705 ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) {
2706 if (params.size() != 2) {
2707 exec_state_ = LDBCommandExecuteResult::Failed(
2708 "<key> and <value> must be specified for the put command");
2709 } else {
2710 key_ = params.at(0);
2711 value_ = params.at(1);
2712 }
2713
2714 if (is_key_hex_) {
2715 key_ = HexToString(key_);
2716 }
2717
2718 if (is_value_hex_) {
2719 value_ = HexToString(value_);
2720 }
2721 create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
2722 }
2723
Help(std::string & ret)2724 void PutCommand::Help(std::string& ret) {
2725 ret.append(" ");
2726 ret.append(PutCommand::Name());
2727 ret.append(" <key> <value> ");
2728 ret.append(" [--" + ARG_TTL + "]");
2729 ret.append("\n");
2730 }
2731
DoCommand()2732 void PutCommand::DoCommand() {
2733 if (!db_) {
2734 assert(GetExecuteState().IsFailed());
2735 return;
2736 }
2737 Status st = db_->Put(WriteOptions(), GetCfHandle(), key_, value_);
2738 if (st.ok()) {
2739 fprintf(stdout, "OK\n");
2740 } else {
2741 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2742 }
2743 }
2744
PrepareOptionsForOpenDB()2745 Options PutCommand::PrepareOptionsForOpenDB() {
2746 Options opt = LDBCommand::PrepareOptionsForOpenDB();
2747 opt.create_if_missing = create_if_missing_;
2748 return opt;
2749 }
2750
2751 // ----------------------------------------------------------------------------
2752
2753 const char* DBQuerierCommand::HELP_CMD = "help";
2754 const char* DBQuerierCommand::GET_CMD = "get";
2755 const char* DBQuerierCommand::PUT_CMD = "put";
2756 const char* DBQuerierCommand::DELETE_CMD = "delete";
2757
DBQuerierCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2758 DBQuerierCommand::DBQuerierCommand(
2759 const std::vector<std::string>& /*params*/,
2760 const std::map<std::string, std::string>& options,
2761 const std::vector<std::string>& flags)
2762 : LDBCommand(
2763 options, flags, false,
2764 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2765
2766 }
2767
Help(std::string & ret)2768 void DBQuerierCommand::Help(std::string& ret) {
2769 ret.append(" ");
2770 ret.append(DBQuerierCommand::Name());
2771 ret.append(" [--" + ARG_TTL + "]");
2772 ret.append("\n");
2773 ret.append(" Starts a REPL shell. Type help for list of available "
2774 "commands.");
2775 ret.append("\n");
2776 }
2777
DoCommand()2778 void DBQuerierCommand::DoCommand() {
2779 if (!db_) {
2780 assert(GetExecuteState().IsFailed());
2781 return;
2782 }
2783
2784 ReadOptions read_options;
2785 WriteOptions write_options;
2786
2787 std::string line;
2788 std::string key;
2789 std::string value;
2790 while (getline(std::cin, line, '\n')) {
2791 // Parse line into std::vector<std::string>
2792 std::vector<std::string> tokens;
2793 size_t pos = 0;
2794 while (true) {
2795 size_t pos2 = line.find(' ', pos);
2796 if (pos2 == std::string::npos) {
2797 break;
2798 }
2799 tokens.push_back(line.substr(pos, pos2-pos));
2800 pos = pos2 + 1;
2801 }
2802 tokens.push_back(line.substr(pos));
2803
2804 const std::string& cmd = tokens[0];
2805
2806 if (cmd == HELP_CMD) {
2807 fprintf(stdout,
2808 "get <key>\n"
2809 "put <key> <value>\n"
2810 "delete <key>\n");
2811 } else if (cmd == DELETE_CMD && tokens.size() == 2) {
2812 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2813 db_->Delete(write_options, GetCfHandle(), Slice(key));
2814 fprintf(stdout, "Successfully deleted %s\n", tokens[1].c_str());
2815 } else if (cmd == PUT_CMD && tokens.size() == 3) {
2816 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2817 value = (is_value_hex_ ? HexToString(tokens[2]) : tokens[2]);
2818 db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value));
2819 fprintf(stdout, "Successfully put %s %s\n",
2820 tokens[1].c_str(), tokens[2].c_str());
2821 } else if (cmd == GET_CMD && tokens.size() == 2) {
2822 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2823 if (db_->Get(read_options, GetCfHandle(), Slice(key), &value).ok()) {
2824 fprintf(stdout, "%s\n", PrintKeyValue(key, value,
2825 is_key_hex_, is_value_hex_).c_str());
2826 } else {
2827 fprintf(stdout, "Not found %s\n", tokens[1].c_str());
2828 }
2829 } else {
2830 fprintf(stdout, "Unknown command %s\n", line.c_str());
2831 }
2832 }
2833 }
2834
2835 // ----------------------------------------------------------------------------
2836
CheckConsistencyCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2837 CheckConsistencyCommand::CheckConsistencyCommand(
2838 const std::vector<std::string>& /*params*/,
2839 const std::map<std::string, std::string>& options,
2840 const std::vector<std::string>& flags)
2841 : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {}
2842
Help(std::string & ret)2843 void CheckConsistencyCommand::Help(std::string& ret) {
2844 ret.append(" ");
2845 ret.append(CheckConsistencyCommand::Name());
2846 ret.append("\n");
2847 }
2848
DoCommand()2849 void CheckConsistencyCommand::DoCommand() {
2850 Options opt = PrepareOptionsForOpenDB();
2851 opt.paranoid_checks = true;
2852 if (!exec_state_.IsNotStarted()) {
2853 return;
2854 }
2855 DB* db;
2856 Status st = DB::OpenForReadOnly(opt, db_path_, &db, false);
2857 delete db;
2858 if (st.ok()) {
2859 fprintf(stdout, "OK\n");
2860 } else {
2861 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2862 }
2863 }
2864
2865 // ----------------------------------------------------------------------------
2866
2867 const std::string CheckPointCommand::ARG_CHECKPOINT_DIR = "checkpoint_dir";
2868
CheckPointCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2869 CheckPointCommand::CheckPointCommand(
2870 const std::vector<std::string>& /*params*/,
2871 const std::map<std::string, std::string>& options,
2872 const std::vector<std::string>& flags)
2873 : LDBCommand(options, flags, false /* is_read_only */,
2874 BuildCmdLineOptions({ARG_CHECKPOINT_DIR})) {
2875 auto itr = options.find(ARG_CHECKPOINT_DIR);
2876 if (itr != options.end()) {
2877 checkpoint_dir_ = itr->second;
2878 }
2879 }
2880
Help(std::string & ret)2881 void CheckPointCommand::Help(std::string& ret) {
2882 ret.append(" ");
2883 ret.append(CheckPointCommand::Name());
2884 ret.append(" [--" + ARG_CHECKPOINT_DIR + "] ");
2885 ret.append("\n");
2886 }
2887
DoCommand()2888 void CheckPointCommand::DoCommand() {
2889 if (!db_) {
2890 assert(GetExecuteState().IsFailed());
2891 return;
2892 }
2893 Checkpoint* checkpoint;
2894 Status status = Checkpoint::Create(db_, &checkpoint);
2895 status = checkpoint->CreateCheckpoint(checkpoint_dir_);
2896 if (status.ok()) {
2897 fprintf(stdout, "OK\n");
2898 } else {
2899 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2900 }
2901 }
2902
2903 // ----------------------------------------------------------------------------
2904
RepairCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2905 RepairCommand::RepairCommand(const std::vector<std::string>& /*params*/,
2906 const std::map<std::string, std::string>& options,
2907 const std::vector<std::string>& flags)
2908 : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {}
2909
Help(std::string & ret)2910 void RepairCommand::Help(std::string& ret) {
2911 ret.append(" ");
2912 ret.append(RepairCommand::Name());
2913 ret.append("\n");
2914 }
2915
DoCommand()2916 void RepairCommand::DoCommand() {
2917 Options options = PrepareOptionsForOpenDB();
2918 options.info_log.reset(new StderrLogger(InfoLogLevel::WARN_LEVEL));
2919 Status status = RepairDB(db_path_, options);
2920 if (status.ok()) {
2921 fprintf(stdout, "OK\n");
2922 } else {
2923 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2924 }
2925 }
2926
2927 // ----------------------------------------------------------------------------
2928
2929 const std::string BackupableCommand::ARG_NUM_THREADS = "num_threads";
2930 const std::string BackupableCommand::ARG_BACKUP_ENV_URI = "backup_env_uri";
2931 const std::string BackupableCommand::ARG_BACKUP_DIR = "backup_dir";
2932 const std::string BackupableCommand::ARG_STDERR_LOG_LEVEL = "stderr_log_level";
2933
BackupableCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2934 BackupableCommand::BackupableCommand(
2935 const std::vector<std::string>& /*params*/,
2936 const std::map<std::string, std::string>& options,
2937 const std::vector<std::string>& flags)
2938 : LDBCommand(options, flags, false /* is_read_only */,
2939 BuildCmdLineOptions({ARG_BACKUP_ENV_URI, ARG_BACKUP_DIR,
2940 ARG_NUM_THREADS, ARG_STDERR_LOG_LEVEL})),
2941 num_threads_(1) {
2942 auto itr = options.find(ARG_NUM_THREADS);
2943 if (itr != options.end()) {
2944 num_threads_ = std::stoi(itr->second);
2945 }
2946 itr = options.find(ARG_BACKUP_ENV_URI);
2947 if (itr != options.end()) {
2948 backup_env_uri_ = itr->second;
2949 }
2950 itr = options.find(ARG_BACKUP_DIR);
2951 if (itr == options.end()) {
2952 exec_state_ = LDBCommandExecuteResult::Failed("--" + ARG_BACKUP_DIR +
2953 ": missing backup directory");
2954 } else {
2955 backup_dir_ = itr->second;
2956 }
2957
2958 itr = options.find(ARG_STDERR_LOG_LEVEL);
2959 if (itr != options.end()) {
2960 int stderr_log_level = std::stoi(itr->second);
2961 if (stderr_log_level < 0 ||
2962 stderr_log_level >= InfoLogLevel::NUM_INFO_LOG_LEVELS) {
2963 exec_state_ = LDBCommandExecuteResult::Failed(
2964 ARG_STDERR_LOG_LEVEL + " must be >= 0 and < " +
2965 std::to_string(InfoLogLevel::NUM_INFO_LOG_LEVELS) + ".");
2966 } else {
2967 logger_.reset(
2968 new StderrLogger(static_cast<InfoLogLevel>(stderr_log_level)));
2969 }
2970 }
2971 }
2972
Help(const std::string & name,std::string & ret)2973 void BackupableCommand::Help(const std::string& name, std::string& ret) {
2974 ret.append(" ");
2975 ret.append(name);
2976 ret.append(" [--" + ARG_BACKUP_ENV_URI + "] ");
2977 ret.append(" [--" + ARG_BACKUP_DIR + "] ");
2978 ret.append(" [--" + ARG_NUM_THREADS + "] ");
2979 ret.append(" [--" + ARG_STDERR_LOG_LEVEL + "=<int (InfoLogLevel)>] ");
2980 ret.append("\n");
2981 }
2982
2983 // ----------------------------------------------------------------------------
2984
BackupCommand(const std::vector<std::string> & params,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)2985 BackupCommand::BackupCommand(const std::vector<std::string>& params,
2986 const std::map<std::string, std::string>& options,
2987 const std::vector<std::string>& flags)
2988 : BackupableCommand(params, options, flags) {}
2989
Help(std::string & ret)2990 void BackupCommand::Help(std::string& ret) {
2991 BackupableCommand::Help(Name(), ret);
2992 }
2993
DoCommand()2994 void BackupCommand::DoCommand() {
2995 BackupEngine* backup_engine;
2996 Status status;
2997 if (!db_) {
2998 assert(GetExecuteState().IsFailed());
2999 return;
3000 }
3001 fprintf(stdout, "open db OK\n");
3002 Env* custom_env = nullptr;
3003 Env::LoadEnv(backup_env_uri_, &custom_env, &backup_env_guard_);
3004 assert(custom_env != nullptr);
3005
3006 BackupableDBOptions backup_options =
3007 BackupableDBOptions(backup_dir_, custom_env);
3008 backup_options.info_log = logger_.get();
3009 backup_options.max_background_operations = num_threads_;
3010 status = BackupEngine::Open(custom_env, backup_options, &backup_engine);
3011 if (status.ok()) {
3012 fprintf(stdout, "open backup engine OK\n");
3013 } else {
3014 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
3015 return;
3016 }
3017 status = backup_engine->CreateNewBackup(db_);
3018 if (status.ok()) {
3019 fprintf(stdout, "create new backup OK\n");
3020 } else {
3021 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
3022 return;
3023 }
3024 }
3025
3026 // ----------------------------------------------------------------------------
3027
RestoreCommand(const std::vector<std::string> & params,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)3028 RestoreCommand::RestoreCommand(
3029 const std::vector<std::string>& params,
3030 const std::map<std::string, std::string>& options,
3031 const std::vector<std::string>& flags)
3032 : BackupableCommand(params, options, flags) {}
3033
Help(std::string & ret)3034 void RestoreCommand::Help(std::string& ret) {
3035 BackupableCommand::Help(Name(), ret);
3036 }
3037
DoCommand()3038 void RestoreCommand::DoCommand() {
3039 Env* custom_env = nullptr;
3040 Env::LoadEnv(backup_env_uri_, &custom_env, &backup_env_guard_);
3041 assert(custom_env != nullptr);
3042
3043 std::unique_ptr<BackupEngineReadOnly> restore_engine;
3044 Status status;
3045 {
3046 BackupableDBOptions opts(backup_dir_, custom_env);
3047 opts.info_log = logger_.get();
3048 opts.max_background_operations = num_threads_;
3049 BackupEngineReadOnly* raw_restore_engine_ptr;
3050 status =
3051 BackupEngineReadOnly::Open(custom_env, opts, &raw_restore_engine_ptr);
3052 if (status.ok()) {
3053 restore_engine.reset(raw_restore_engine_ptr);
3054 }
3055 }
3056 if (status.ok()) {
3057 fprintf(stdout, "open restore engine OK\n");
3058 status = restore_engine->RestoreDBFromLatestBackup(db_path_, db_path_);
3059 }
3060 if (status.ok()) {
3061 fprintf(stdout, "restore from backup OK\n");
3062 } else {
3063 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
3064 }
3065 }
3066
3067 // ----------------------------------------------------------------------------
3068
3069 namespace {
3070
DumpSstFile(Options options,std::string filename,bool output_hex,bool show_properties)3071 void DumpSstFile(Options options, std::string filename, bool output_hex,
3072 bool show_properties) {
3073 std::string from_key;
3074 std::string to_key;
3075 if (filename.length() <= 4 ||
3076 filename.rfind(".sst") != filename.length() - 4) {
3077 std::cout << "Invalid sst file name." << std::endl;
3078 return;
3079 }
3080 // no verification
3081 // TODO: add support for decoding blob indexes in ldb as well
3082 ROCKSDB_NAMESPACE::SstFileDumper dumper(
3083 options, filename, /* verify_checksum */ false, output_hex,
3084 /* decode_blob_index */ false);
3085 Status st = dumper.ReadSequential(true, std::numeric_limits<uint64_t>::max(),
3086 false, // has_from
3087 from_key, false, // has_to
3088 to_key);
3089 if (!st.ok()) {
3090 std::cerr << "Error in reading SST file " << filename << st.ToString()
3091 << std::endl;
3092 return;
3093 }
3094
3095 if (show_properties) {
3096 const ROCKSDB_NAMESPACE::TableProperties* table_properties;
3097
3098 std::shared_ptr<const ROCKSDB_NAMESPACE::TableProperties>
3099 table_properties_from_reader;
3100 st = dumper.ReadTableProperties(&table_properties_from_reader);
3101 if (!st.ok()) {
3102 std::cerr << filename << ": " << st.ToString()
3103 << ". Try to use initial table properties" << std::endl;
3104 table_properties = dumper.GetInitTableProperties();
3105 } else {
3106 table_properties = table_properties_from_reader.get();
3107 }
3108 if (table_properties != nullptr) {
3109 std::cout << std::endl << "Table Properties:" << std::endl;
3110 std::cout << table_properties->ToString("\n") << std::endl;
3111 }
3112 }
3113 }
3114
3115 } // namespace
3116
DBFileDumperCommand(const std::vector<std::string> &,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)3117 DBFileDumperCommand::DBFileDumperCommand(
3118 const std::vector<std::string>& /*params*/,
3119 const std::map<std::string, std::string>& options,
3120 const std::vector<std::string>& flags)
3121 : LDBCommand(options, flags, true, BuildCmdLineOptions({})) {}
3122
Help(std::string & ret)3123 void DBFileDumperCommand::Help(std::string& ret) {
3124 ret.append(" ");
3125 ret.append(DBFileDumperCommand::Name());
3126 ret.append("\n");
3127 }
3128
DoCommand()3129 void DBFileDumperCommand::DoCommand() {
3130 if (!db_) {
3131 assert(GetExecuteState().IsFailed());
3132 return;
3133 }
3134 Status s;
3135
3136 std::cout << "Manifest File" << std::endl;
3137 std::cout << "==============================" << std::endl;
3138 std::string manifest_filename;
3139 s = ReadFileToString(db_->GetEnv(), CurrentFileName(db_->GetName()),
3140 &manifest_filename);
3141 if (!s.ok() || manifest_filename.empty() ||
3142 manifest_filename.back() != '\n') {
3143 std::cerr << "Error when reading CURRENT file "
3144 << CurrentFileName(db_->GetName()) << std::endl;
3145 }
3146 // remove the trailing '\n'
3147 manifest_filename.resize(manifest_filename.size() - 1);
3148 std::string manifest_filepath = db_->GetName() + "/" + manifest_filename;
3149 std::cout << manifest_filepath << std::endl;
3150 DumpManifestFile(options_, manifest_filepath, false, false, false);
3151 std::cout << std::endl;
3152
3153 std::cout << "SST Files" << std::endl;
3154 std::cout << "==============================" << std::endl;
3155 std::vector<LiveFileMetaData> metadata;
3156 db_->GetLiveFilesMetaData(&metadata);
3157 for (auto& fileMetadata : metadata) {
3158 std::string filename = fileMetadata.db_path + fileMetadata.name;
3159 std::cout << filename << " level:" << fileMetadata.level << std::endl;
3160 std::cout << "------------------------------" << std::endl;
3161 DumpSstFile(options_, filename, false, true);
3162 std::cout << std::endl;
3163 }
3164 std::cout << std::endl;
3165
3166 std::cout << "Write Ahead Log Files" << std::endl;
3167 std::cout << "==============================" << std::endl;
3168 ROCKSDB_NAMESPACE::VectorLogPtr wal_files;
3169 s = db_->GetSortedWalFiles(wal_files);
3170 if (!s.ok()) {
3171 std::cerr << "Error when getting WAL files" << std::endl;
3172 } else {
3173 for (auto& wal : wal_files) {
3174 // TODO(qyang): option.wal_dir should be passed into ldb command
3175 std::string filename = db_->GetOptions().wal_dir + wal->PathName();
3176 std::cout << filename << std::endl;
3177 // TODO(myabandeh): allow configuring is_write_commited
3178 DumpWalFile(options_, filename, true, true, true /* is_write_commited */,
3179 &exec_state_);
3180 }
3181 }
3182 }
3183
Help(std::string & ret)3184 void WriteExternalSstFilesCommand::Help(std::string& ret) {
3185 ret.append(" ");
3186 ret.append(WriteExternalSstFilesCommand::Name());
3187 ret.append(" <output_sst_path>");
3188 ret.append("\n");
3189 }
3190
WriteExternalSstFilesCommand(const std::vector<std::string> & params,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)3191 WriteExternalSstFilesCommand::WriteExternalSstFilesCommand(
3192 const std::vector<std::string>& params,
3193 const std::map<std::string, std::string>& options,
3194 const std::vector<std::string>& flags)
3195 : LDBCommand(
3196 options, flags, false /* is_read_only */,
3197 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
3198 ARG_TO, ARG_CREATE_IF_MISSING})) {
3199 create_if_missing_ =
3200 IsFlagPresent(flags, ARG_CREATE_IF_MISSING) ||
3201 ParseBooleanOption(options, ARG_CREATE_IF_MISSING, false);
3202 if (params.size() != 1) {
3203 exec_state_ = LDBCommandExecuteResult::Failed(
3204 "output SST file path must be specified");
3205 } else {
3206 output_sst_path_ = params.at(0);
3207 }
3208 }
3209
DoCommand()3210 void WriteExternalSstFilesCommand::DoCommand() {
3211 if (!db_) {
3212 assert(GetExecuteState().IsFailed());
3213 return;
3214 }
3215 ColumnFamilyHandle* cfh = GetCfHandle();
3216 SstFileWriter sst_file_writer(EnvOptions(), db_->GetOptions(), cfh);
3217 Status status = sst_file_writer.Open(output_sst_path_);
3218 if (!status.ok()) {
3219 exec_state_ = LDBCommandExecuteResult::Failed("failed to open SST file: " +
3220 status.ToString());
3221 return;
3222 }
3223
3224 int bad_lines = 0;
3225 std::string line;
3226 std::ifstream ifs_stdin("/dev/stdin");
3227 std::istream* istream_p = ifs_stdin.is_open() ? &ifs_stdin : &std::cin;
3228 while (getline(*istream_p, line, '\n')) {
3229 std::string key;
3230 std::string value;
3231 if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) {
3232 status = sst_file_writer.Put(key, value);
3233 if (!status.ok()) {
3234 exec_state_ = LDBCommandExecuteResult::Failed(
3235 "failed to write record to file: " + status.ToString());
3236 return;
3237 }
3238 } else if (0 == line.find("Keys in range:")) {
3239 // ignore this line
3240 } else if (0 == line.find("Created bg thread 0x")) {
3241 // ignore this line
3242 } else {
3243 bad_lines++;
3244 }
3245 }
3246
3247 status = sst_file_writer.Finish();
3248 if (!status.ok()) {
3249 exec_state_ = LDBCommandExecuteResult::Failed(
3250 "Failed to finish writing to file: " + status.ToString());
3251 return;
3252 }
3253
3254 if (bad_lines > 0) {
3255 fprintf(stderr, "Warning: %d bad lines ignored.\n", bad_lines);
3256 }
3257 exec_state_ = LDBCommandExecuteResult::Succeed(
3258 "external SST file written to " + output_sst_path_);
3259 }
3260
PrepareOptionsForOpenDB()3261 Options WriteExternalSstFilesCommand::PrepareOptionsForOpenDB() {
3262 Options opt = LDBCommand::PrepareOptionsForOpenDB();
3263 opt.create_if_missing = create_if_missing_;
3264 return opt;
3265 }
3266
3267 const std::string IngestExternalSstFilesCommand::ARG_MOVE_FILES = "move_files";
3268 const std::string IngestExternalSstFilesCommand::ARG_SNAPSHOT_CONSISTENCY =
3269 "snapshot_consistency";
3270 const std::string IngestExternalSstFilesCommand::ARG_ALLOW_GLOBAL_SEQNO =
3271 "allow_global_seqno";
3272 const std::string IngestExternalSstFilesCommand::ARG_ALLOW_BLOCKING_FLUSH =
3273 "allow_blocking_flush";
3274 const std::string IngestExternalSstFilesCommand::ARG_INGEST_BEHIND =
3275 "ingest_behind";
3276 const std::string IngestExternalSstFilesCommand::ARG_WRITE_GLOBAL_SEQNO =
3277 "write_global_seqno";
3278
Help(std::string & ret)3279 void IngestExternalSstFilesCommand::Help(std::string& ret) {
3280 ret.append(" ");
3281 ret.append(IngestExternalSstFilesCommand::Name());
3282 ret.append(" <input_sst_path>");
3283 ret.append(" [--" + ARG_MOVE_FILES + "] ");
3284 ret.append(" [--" + ARG_SNAPSHOT_CONSISTENCY + "] ");
3285 ret.append(" [--" + ARG_ALLOW_GLOBAL_SEQNO + "] ");
3286 ret.append(" [--" + ARG_ALLOW_BLOCKING_FLUSH + "] ");
3287 ret.append(" [--" + ARG_INGEST_BEHIND + "] ");
3288 ret.append(" [--" + ARG_WRITE_GLOBAL_SEQNO + "] ");
3289 ret.append("\n");
3290 }
3291
IngestExternalSstFilesCommand(const std::vector<std::string> & params,const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)3292 IngestExternalSstFilesCommand::IngestExternalSstFilesCommand(
3293 const std::vector<std::string>& params,
3294 const std::map<std::string, std::string>& options,
3295 const std::vector<std::string>& flags)
3296 : LDBCommand(
3297 options, flags, false /* is_read_only */,
3298 BuildCmdLineOptions({ARG_MOVE_FILES, ARG_SNAPSHOT_CONSISTENCY,
3299 ARG_ALLOW_GLOBAL_SEQNO, ARG_CREATE_IF_MISSING,
3300 ARG_ALLOW_BLOCKING_FLUSH, ARG_INGEST_BEHIND,
3301 ARG_WRITE_GLOBAL_SEQNO})),
3302 move_files_(false),
3303 snapshot_consistency_(true),
3304 allow_global_seqno_(true),
3305 allow_blocking_flush_(true),
3306 ingest_behind_(false),
3307 write_global_seqno_(true) {
3308 create_if_missing_ =
3309 IsFlagPresent(flags, ARG_CREATE_IF_MISSING) ||
3310 ParseBooleanOption(options, ARG_CREATE_IF_MISSING, false);
3311 move_files_ = IsFlagPresent(flags, ARG_MOVE_FILES) ||
3312 ParseBooleanOption(options, ARG_MOVE_FILES, false);
3313 snapshot_consistency_ =
3314 IsFlagPresent(flags, ARG_SNAPSHOT_CONSISTENCY) ||
3315 ParseBooleanOption(options, ARG_SNAPSHOT_CONSISTENCY, true);
3316 allow_global_seqno_ =
3317 IsFlagPresent(flags, ARG_ALLOW_GLOBAL_SEQNO) ||
3318 ParseBooleanOption(options, ARG_ALLOW_GLOBAL_SEQNO, true);
3319 allow_blocking_flush_ =
3320 IsFlagPresent(flags, ARG_ALLOW_BLOCKING_FLUSH) ||
3321 ParseBooleanOption(options, ARG_ALLOW_BLOCKING_FLUSH, true);
3322 ingest_behind_ = IsFlagPresent(flags, ARG_INGEST_BEHIND) ||
3323 ParseBooleanOption(options, ARG_INGEST_BEHIND, false);
3324 write_global_seqno_ =
3325 IsFlagPresent(flags, ARG_WRITE_GLOBAL_SEQNO) ||
3326 ParseBooleanOption(options, ARG_WRITE_GLOBAL_SEQNO, true);
3327
3328 if (allow_global_seqno_) {
3329 if (!write_global_seqno_) {
3330 fprintf(stderr,
3331 "Warning: not writing global_seqno to the ingested SST can\n"
3332 "prevent older versions of RocksDB from being able to open it\n");
3333 }
3334 } else {
3335 if (write_global_seqno_) {
3336 exec_state_ = LDBCommandExecuteResult::Failed(
3337 "ldb cannot write global_seqno to the ingested SST when global_seqno "
3338 "is not allowed");
3339 }
3340 }
3341
3342 if (params.size() != 1) {
3343 exec_state_ =
3344 LDBCommandExecuteResult::Failed("input SST path must be specified");
3345 } else {
3346 input_sst_path_ = params.at(0);
3347 }
3348 }
3349
DoCommand()3350 void IngestExternalSstFilesCommand::DoCommand() {
3351 if (!db_) {
3352 assert(GetExecuteState().IsFailed());
3353 return;
3354 }
3355 if (GetExecuteState().IsFailed()) {
3356 return;
3357 }
3358 ColumnFamilyHandle* cfh = GetCfHandle();
3359 IngestExternalFileOptions ifo;
3360 ifo.move_files = move_files_;
3361 ifo.snapshot_consistency = snapshot_consistency_;
3362 ifo.allow_global_seqno = allow_global_seqno_;
3363 ifo.allow_blocking_flush = allow_blocking_flush_;
3364 ifo.ingest_behind = ingest_behind_;
3365 ifo.write_global_seqno = write_global_seqno_;
3366 Status status = db_->IngestExternalFile(cfh, {input_sst_path_}, ifo);
3367 if (!status.ok()) {
3368 exec_state_ = LDBCommandExecuteResult::Failed(
3369 "failed to ingest external SST: " + status.ToString());
3370 } else {
3371 exec_state_ =
3372 LDBCommandExecuteResult::Succeed("external SST files ingested");
3373 }
3374 }
3375
PrepareOptionsForOpenDB()3376 Options IngestExternalSstFilesCommand::PrepareOptionsForOpenDB() {
3377 Options opt = LDBCommand::PrepareOptionsForOpenDB();
3378 opt.create_if_missing = create_if_missing_;
3379 return opt;
3380 }
3381
ListFileRangeDeletesCommand(const std::map<std::string,std::string> & options,const std::vector<std::string> & flags)3382 ListFileRangeDeletesCommand::ListFileRangeDeletesCommand(
3383 const std::map<std::string, std::string>& options,
3384 const std::vector<std::string>& flags)
3385 : LDBCommand(options, flags, true, BuildCmdLineOptions({ARG_MAX_KEYS})) {
3386 std::map<std::string, std::string>::const_iterator itr =
3387 options.find(ARG_MAX_KEYS);
3388 if (itr != options.end()) {
3389 try {
3390 #if defined(CYGWIN)
3391 max_keys_ = strtol(itr->second.c_str(), 0, 10);
3392 #else
3393 max_keys_ = std::stoi(itr->second);
3394 #endif
3395 } catch (const std::invalid_argument&) {
3396 exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
3397 " has an invalid value");
3398 } catch (const std::out_of_range&) {
3399 exec_state_ = LDBCommandExecuteResult::Failed(
3400 ARG_MAX_KEYS + " has a value out-of-range");
3401 }
3402 }
3403 }
3404
Help(std::string & ret)3405 void ListFileRangeDeletesCommand::Help(std::string& ret) {
3406 ret.append(" ");
3407 ret.append(ListFileRangeDeletesCommand::Name());
3408 ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
3409 ret.append(" : print tombstones in SST files.\n");
3410 }
3411
DoCommand()3412 void ListFileRangeDeletesCommand::DoCommand() {
3413 if (!db_) {
3414 assert(GetExecuteState().IsFailed());
3415 return;
3416 }
3417
3418 DBImpl* db_impl = static_cast_with_check<DBImpl, DB>(db_->GetRootDB());
3419
3420 std::string out_str;
3421
3422 Status st =
3423 db_impl->TablesRangeTombstoneSummary(GetCfHandle(), max_keys_, &out_str);
3424 if (st.ok()) {
3425 TEST_SYNC_POINT_CALLBACK(
3426 "ListFileRangeDeletesCommand::DoCommand:BeforePrint", &out_str);
3427 fprintf(stdout, "%s\n", out_str.c_str());
3428 } else {
3429 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
3430 }
3431 }
3432
3433 } // namespace ROCKSDB_NAMESPACE
3434 #endif // ROCKSDB_LITE
3435