1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2 //  This source code is licensed under both the GPLv2 (found in the
3 //  COPYING file in the root directory) and Apache 2.0 License
4 //  (found in the LICENSE.Apache file in the root directory).
5 
6 #pragma once
7 
8 #ifndef ROCKSDB_LITE
9 
10 #include <functional>
11 #include <string>
12 #include <vector>
13 #include "rocksdb/db.h"
14 #include "rocksdb/status.h"
15 #include "rocksdb/utilities/stackable_db.h"
16 
17 namespace ROCKSDB_NAMESPACE {
18 
19 namespace blob_db {
20 
21 // A wrapped database which puts values of KV pairs in a separate log
22 // and store location to the log in the underlying DB.
23 //
24 // The factory needs to be moved to include/rocksdb/utilities to allow
25 // users to use blob DB.
26 
27 struct BlobDBOptions {
28   // name of the directory under main db, where blobs will be stored.
29   // default is "blob_dir"
30   std::string blob_dir = "blob_dir";
31 
32   // whether the blob_dir path is relative or absolute.
33   bool path_relative = true;
34 
35   // When max_db_size is reached, evict blob files to free up space
36   // instead of returnning NoSpace error on write. Blob files will be
37   // evicted from oldest to newest, based on file creation time.
38   bool is_fifo = false;
39 
40   // Maximum size of the database (including SST files and blob files).
41   //
42   // Default: 0 (no limits)
43   uint64_t max_db_size = 0;
44 
45   // a new bucket is opened, for ttl_range. So if ttl_range is 600seconds
46   // (10 minutes), and the first bucket starts at 1471542000
47   // then the blob buckets will be
48   // first bucket is 1471542000 - 1471542600
49   // second bucket is 1471542600 - 1471543200
50   // and so on
51   uint64_t ttl_range_secs = 3600;
52 
53   // The smallest value to store in blob log. Values smaller than this threshold
54   // will be inlined in base DB together with the key.
55   uint64_t min_blob_size = 0;
56 
57   // Allows OS to incrementally sync blob files to disk for every
58   // bytes_per_sync bytes written. Users shouldn't rely on it for
59   // persistency guarantee.
60   uint64_t bytes_per_sync = 512 * 1024;
61 
62   // the target size of each blob file. File will become immutable
63   // after it exceeds that size
64   uint64_t blob_file_size = 256 * 1024 * 1024;
65 
66   // what compression to use for Blob's
67   CompressionType compression = kNoCompression;
68 
69   // If enabled, BlobDB cleans up stale blobs in non-TTL files during compaction
70   // by rewriting the remaining live blobs to new files.
71   bool enable_garbage_collection = false;
72 
73   // The cutoff in terms of blob file age for garbage collection. Blobs in
74   // the oldest N non-TTL blob files will be rewritten when encountered during
75   // compaction, where N = garbage_collection_cutoff * number_of_non_TTL_files.
76   double garbage_collection_cutoff = 0.25;
77 
78   // Disable all background job. Used for test only.
79   bool disable_background_tasks = false;
80 
81   void Dump(Logger* log) const;
82 };
83 
84 class BlobDB : public StackableDB {
85  public:
86   using ROCKSDB_NAMESPACE::StackableDB::Put;
87   virtual Status Put(const WriteOptions& options, const Slice& key,
88                      const Slice& value) override = 0;
Put(const WriteOptions & options,ColumnFamilyHandle * column_family,const Slice & key,const Slice & value)89   virtual Status Put(const WriteOptions& options,
90                      ColumnFamilyHandle* column_family, const Slice& key,
91                      const Slice& value) override {
92     if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
93       return Status::NotSupported(
94           "Blob DB doesn't support non-default column family.");
95     }
96     return Put(options, key, value);
97   }
98 
99   using ROCKSDB_NAMESPACE::StackableDB::Delete;
Delete(const WriteOptions & options,ColumnFamilyHandle * column_family,const Slice & key)100   virtual Status Delete(const WriteOptions& options,
101                         ColumnFamilyHandle* column_family,
102                         const Slice& key) override {
103     if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
104       return Status::NotSupported(
105           "Blob DB doesn't support non-default column family.");
106     }
107     assert(db_ != nullptr);
108     return db_->Delete(options, column_family, key);
109   }
110 
111   virtual Status PutWithTTL(const WriteOptions& options, const Slice& key,
112                             const Slice& value, uint64_t ttl) = 0;
PutWithTTL(const WriteOptions & options,ColumnFamilyHandle * column_family,const Slice & key,const Slice & value,uint64_t ttl)113   virtual Status PutWithTTL(const WriteOptions& options,
114                             ColumnFamilyHandle* column_family, const Slice& key,
115                             const Slice& value, uint64_t ttl) {
116     if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
117       return Status::NotSupported(
118           "Blob DB doesn't support non-default column family.");
119     }
120     return PutWithTTL(options, key, value, ttl);
121   }
122 
123   // Put with expiration. Key with expiration time equal to
124   // std::numeric_limits<uint64_t>::max() means the key don't expire.
125   virtual Status PutUntil(const WriteOptions& options, const Slice& key,
126                           const Slice& value, uint64_t expiration) = 0;
PutUntil(const WriteOptions & options,ColumnFamilyHandle * column_family,const Slice & key,const Slice & value,uint64_t expiration)127   virtual Status PutUntil(const WriteOptions& options,
128                           ColumnFamilyHandle* column_family, const Slice& key,
129                           const Slice& value, uint64_t expiration) {
130     if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
131       return Status::NotSupported(
132           "Blob DB doesn't support non-default column family.");
133     }
134     return PutUntil(options, key, value, expiration);
135   }
136 
137   using ROCKSDB_NAMESPACE::StackableDB::Get;
138   virtual Status Get(const ReadOptions& options,
139                      ColumnFamilyHandle* column_family, const Slice& key,
140                      PinnableSlice* value) override = 0;
141 
142   // Get value and expiration.
143   virtual Status Get(const ReadOptions& options,
144                      ColumnFamilyHandle* column_family, const Slice& key,
145                      PinnableSlice* value, uint64_t* expiration) = 0;
Get(const ReadOptions & options,const Slice & key,PinnableSlice * value,uint64_t * expiration)146   virtual Status Get(const ReadOptions& options, const Slice& key,
147                      PinnableSlice* value, uint64_t* expiration) {
148     return Get(options, DefaultColumnFamily(), key, value, expiration);
149   }
150 
151   using ROCKSDB_NAMESPACE::StackableDB::MultiGet;
152   virtual std::vector<Status> MultiGet(
153       const ReadOptions& options,
154       const std::vector<Slice>& keys,
155       std::vector<std::string>* values) override = 0;
MultiGet(const ReadOptions & options,const std::vector<ColumnFamilyHandle * > & column_families,const std::vector<Slice> & keys,std::vector<std::string> * values)156   virtual std::vector<Status> MultiGet(
157       const ReadOptions& options,
158       const std::vector<ColumnFamilyHandle*>& column_families,
159       const std::vector<Slice>& keys,
160       std::vector<std::string>* values) override {
161     for (auto column_family : column_families) {
162       if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
163         return std::vector<Status>(
164             column_families.size(),
165             Status::NotSupported(
166                 "Blob DB doesn't support non-default column family."));
167       }
168     }
169     return MultiGet(options, keys, values);
170   }
171   virtual void MultiGet(const ReadOptions& /*options*/,
172                         ColumnFamilyHandle* /*column_family*/,
173                         const size_t num_keys, const Slice* /*keys*/,
174                         PinnableSlice* /*values*/, Status* statuses,
175                         const bool /*sorted_input*/ = false) override {
176     for (size_t i = 0; i < num_keys; ++i) {
177       statuses[i] = Status::NotSupported(
178           "Blob DB doesn't support batched MultiGet");
179     }
180   }
181 
182   using ROCKSDB_NAMESPACE::StackableDB::SingleDelete;
SingleDelete(const WriteOptions &,ColumnFamilyHandle *,const Slice &)183   virtual Status SingleDelete(const WriteOptions& /*wopts*/,
184                               ColumnFamilyHandle* /*column_family*/,
185                               const Slice& /*key*/) override {
186     return Status::NotSupported("Not supported operation in blob db.");
187   }
188 
189   using ROCKSDB_NAMESPACE::StackableDB::Merge;
Merge(const WriteOptions &,ColumnFamilyHandle *,const Slice &,const Slice &)190   virtual Status Merge(const WriteOptions& /*options*/,
191                        ColumnFamilyHandle* /*column_family*/,
192                        const Slice& /*key*/, const Slice& /*value*/) override {
193     return Status::NotSupported("Not supported operation in blob db.");
194   }
195 
196   virtual Status Write(const WriteOptions& opts,
197                        WriteBatch* updates) override = 0;
198   using ROCKSDB_NAMESPACE::StackableDB::NewIterator;
199   virtual Iterator* NewIterator(const ReadOptions& options) override = 0;
NewIterator(const ReadOptions & options,ColumnFamilyHandle * column_family)200   virtual Iterator* NewIterator(const ReadOptions& options,
201                                 ColumnFamilyHandle* column_family) override {
202     if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
203       // Blob DB doesn't support non-default column family.
204       return nullptr;
205     }
206     return NewIterator(options);
207   }
208 
209   Status CompactFiles(
210       const CompactionOptions& compact_options,
211       const std::vector<std::string>& input_file_names, const int output_level,
212       const int output_path_id = -1,
213       std::vector<std::string>* const output_file_names = nullptr,
214       CompactionJobInfo* compaction_job_info = nullptr) override = 0;
215   Status CompactFiles(
216       const CompactionOptions& compact_options,
217       ColumnFamilyHandle* column_family,
218       const std::vector<std::string>& input_file_names, const int output_level,
219       const int output_path_id = -1,
220       std::vector<std::string>* const output_file_names = nullptr,
221       CompactionJobInfo* compaction_job_info = nullptr) override {
222     if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
223       return Status::NotSupported(
224           "Blob DB doesn't support non-default column family.");
225     }
226 
227     return CompactFiles(compact_options, input_file_names, output_level,
228                         output_path_id, output_file_names, compaction_job_info);
229   }
230 
231   using ROCKSDB_NAMESPACE::StackableDB::Close;
232   virtual Status Close() override = 0;
233 
234   // Opening blob db.
235   static Status Open(const Options& options, const BlobDBOptions& bdb_options,
236                      const std::string& dbname, BlobDB** blob_db);
237 
238   static Status Open(const DBOptions& db_options,
239                      const BlobDBOptions& bdb_options,
240                      const std::string& dbname,
241                      const std::vector<ColumnFamilyDescriptor>& column_families,
242                      std::vector<ColumnFamilyHandle*>* handles,
243                      BlobDB** blob_db);
244 
245   virtual BlobDBOptions GetBlobDBOptions() const = 0;
246 
247   virtual Status SyncBlobFiles() = 0;
248 
~BlobDB()249   virtual ~BlobDB() {}
250 
251  protected:
252   explicit BlobDB();
253 };
254 
255 // Destroy the content of the database.
256 Status DestroyBlobDB(const std::string& dbname, const Options& options,
257                      const BlobDBOptions& bdb_options);
258 
259 }  // namespace blob_db
260 }  // namespace ROCKSDB_NAMESPACE
261 #endif  // ROCKSDB_LITE
262