1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 // This source code is licensed under both the GPLv2 (found in the 3 // COPYING file in the root directory) and Apache 2.0 License 4 // (found in the LICENSE.Apache file in the root directory). 5 6 #pragma once 7 8 #ifndef ROCKSDB_LITE 9 10 #include <string> 11 12 #include "port/port.h" 13 14 #include "db/compaction/compaction.h" 15 #include "db/error_handler.h" 16 #include "file/delete_scheduler.h" 17 #include "rocksdb/file_system.h" 18 #include "rocksdb/sst_file_manager.h" 19 20 namespace ROCKSDB_NAMESPACE { 21 22 class Env; 23 class Logger; 24 25 // SstFileManager is used to track SST files in the DB and control there 26 // deletion rate. 27 // All SstFileManager public functions are thread-safe. 28 class SstFileManagerImpl : public SstFileManager { 29 public: 30 explicit SstFileManagerImpl(Env* env, std::shared_ptr<FileSystem> fs, 31 std::shared_ptr<Logger> logger, 32 int64_t rate_bytes_per_sec, 33 double max_trash_db_ratio, 34 uint64_t bytes_max_delete_chunk); 35 36 ~SstFileManagerImpl(); 37 38 // DB will call OnAddFile whenever a new sst file is added. 39 Status OnAddFile(const std::string& file_path, bool compaction = false); 40 41 // Overload where size of the file is provided by the caller rather than 42 // queried from the filesystem. This is an optimization. 43 Status OnAddFile(const std::string& file_path, uint64_t file_size, 44 bool compaction); 45 46 // DB will call OnDeleteFile whenever an sst file is deleted. 47 Status OnDeleteFile(const std::string& file_path); 48 49 // DB will call OnMoveFile whenever an sst file is move to a new path. 50 Status OnMoveFile(const std::string& old_path, const std::string& new_path, 51 uint64_t* file_size = nullptr); 52 53 // Update the maximum allowed space that should be used by RocksDB, if 54 // the total size of the SST files exceeds max_allowed_space, writes to 55 // RocksDB will fail. 56 // 57 // Setting max_allowed_space to 0 will disable this feature, maximum allowed 58 // space will be infinite (Default value). 59 // 60 // thread-safe. 61 void SetMaxAllowedSpaceUsage(uint64_t max_allowed_space) override; 62 63 void SetCompactionBufferSize(uint64_t compaction_buffer_size) override; 64 65 // Return true if the total size of SST files exceeded the maximum allowed 66 // space usage. 67 // 68 // thread-safe. 69 bool IsMaxAllowedSpaceReached() override; 70 71 bool IsMaxAllowedSpaceReachedIncludingCompactions() override; 72 73 // Returns true is there is enough (approximate) space for the specified 74 // compaction. Space is approximate because this function conservatively 75 // estimates how much space is currently being used by compactions (i.e. 76 // if a compaction has started, this function bumps the used space by 77 // the full compaction size). 78 bool EnoughRoomForCompaction(ColumnFamilyData* cfd, 79 const std::vector<CompactionInputFiles>& inputs, 80 Status bg_error); 81 82 // Bookkeeping so total_file_sizes_ goes back to normal after compaction 83 // finishes 84 void OnCompactionCompletion(Compaction* c); 85 86 uint64_t GetCompactionsReservedSize(); 87 88 // Return the total size of all tracked files. 89 uint64_t GetTotalSize() override; 90 91 // Return a map containing all tracked files and there corresponding sizes. 92 std::unordered_map<std::string, uint64_t> GetTrackedFiles() override; 93 94 // Return delete rate limit in bytes per second. 95 virtual int64_t GetDeleteRateBytesPerSecond() override; 96 97 // Update the delete rate limit in bytes per second. 98 virtual void SetDeleteRateBytesPerSecond(int64_t delete_rate) override; 99 100 // Return trash/DB size ratio where new files will be deleted immediately 101 virtual double GetMaxTrashDBRatio() override; 102 103 // Update trash/DB size ratio where new files will be deleted immediately 104 virtual void SetMaxTrashDBRatio(double ratio) override; 105 106 // Return the total size of trash files 107 uint64_t GetTotalTrashSize() override; 108 109 // Called by each DB instance using this sst file manager to reserve 110 // disk buffer space for recovery from out of space errors 111 void ReserveDiskBuffer(uint64_t buffer, const std::string& path); 112 113 // Set a flag upon encountering disk full. May enqueue the ErrorHandler 114 // instance for background polling and recovery 115 void StartErrorRecovery(ErrorHandler* db, Status bg_error); 116 117 // Remove the given Errorhandler instance from the recovery queue. Its 118 // not guaranteed 119 bool CancelErrorRecovery(ErrorHandler* db); 120 121 // Mark file as trash and schedule it's deletion. If force_bg is set, it 122 // forces the file to be deleting in the background regardless of DB size, 123 // except when rate limited delete is disabled 124 virtual Status ScheduleFileDeletion(const std::string& file_path, 125 const std::string& dir_to_sync, 126 const bool force_bg = false); 127 128 // Wait for all files being deleteing in the background to finish or for 129 // destructor to be called. 130 virtual void WaitForEmptyTrash(); 131 delete_scheduler()132 DeleteScheduler* delete_scheduler() { return &delete_scheduler_; } 133 134 // Stop the error recovery background thread. This should be called only 135 // once in the object's lifetime, and before the destructor 136 void Close(); 137 138 private: 139 // REQUIRES: mutex locked 140 void OnAddFileImpl(const std::string& file_path, uint64_t file_size, 141 bool compaction); 142 // REQUIRES: mutex locked 143 void OnDeleteFileImpl(const std::string& file_path); 144 145 void ClearError(); CheckFreeSpace()146 bool CheckFreeSpace() { 147 return bg_err_.severity() == Status::Severity::kSoftError; 148 } 149 150 Env* env_; 151 std::shared_ptr<FileSystem> fs_; 152 std::shared_ptr<Logger> logger_; 153 // Mutex to protect tracked_files_, total_files_size_ 154 port::Mutex mu_; 155 // The summation of the sizes of all files in tracked_files_ map 156 uint64_t total_files_size_; 157 // The summation of all output files of in-progress compactions 158 uint64_t in_progress_files_size_; 159 // Compactions should only execute if they can leave at least 160 // this amount of buffer space for logs and flushes 161 uint64_t compaction_buffer_size_; 162 // Estimated size of the current ongoing compactions 163 uint64_t cur_compactions_reserved_size_; 164 // A map containing all tracked files and there sizes 165 // file_path => file_size 166 std::unordered_map<std::string, uint64_t> tracked_files_; 167 // A set of files belonging to in-progress compactions 168 std::unordered_set<std::string> in_progress_files_; 169 // The maximum allowed space (in bytes) for sst files. 170 uint64_t max_allowed_space_; 171 // DeleteScheduler used to throttle file deletition. 172 DeleteScheduler delete_scheduler_; 173 port::CondVar cv_; 174 // Flag to force error recovery thread to exit 175 bool closing_; 176 // Background error recovery thread 177 std::unique_ptr<port::Thread> bg_thread_; 178 // A path in the filesystem corresponding to this SFM. This is used for 179 // calling Env::GetFreeSpace. Posix requires a path in the filesystem 180 std::string path_; 181 // Save the current background error 182 Status bg_err_; 183 // Amount of free disk headroom before allowing recovery from hard errors 184 uint64_t reserved_disk_buffer_; 185 // For soft errors, amount of free disk space before we can allow 186 // compactions to run full throttle. If disk space is below this trigger, 187 // compactions will be gated by free disk space > input size 188 uint64_t free_space_trigger_; 189 // List of database error handler instances tracked by this sst file manager 190 std::list<ErrorHandler*> error_handler_list_; 191 // Pointer to ErrorHandler instance that is currently processing recovery 192 ErrorHandler* cur_instance_; 193 }; 194 195 } // namespace ROCKSDB_NAMESPACE 196 197 #endif // ROCKSDB_LITE 198