1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2 //  This source code is licensed under both the GPLv2 (found in the
3 //  COPYING file in the root directory) and Apache 2.0 License
4 //  (found in the LICENSE.Apache file in the root directory).
5 //
6 // Log format information shared by reader and writer.
7 
8 #pragma once
9 
10 #ifndef ROCKSDB_LITE
11 
12 #include <limits>
13 #include <memory>
14 #include <utility>
15 
16 #include "rocksdb/options.h"
17 #include "rocksdb/slice.h"
18 #include "rocksdb/status.h"
19 #include "rocksdb/types.h"
20 
21 namespace ROCKSDB_NAMESPACE {
22 namespace blob_db {
23 
24 constexpr uint32_t kMagicNumber = 2395959;  // 0x00248f37
25 constexpr uint32_t kVersion1 = 1;
26 constexpr uint64_t kNoExpiration = std::numeric_limits<uint64_t>::max();
27 
28 using ExpirationRange = std::pair<uint64_t, uint64_t>;
29 
30 // Format of blob log file header (30 bytes):
31 //
32 //    +--------------+---------+---------+-------+-------------+-------------------+
33 //    | magic number | version |  cf id  | flags | compression | expiration range  |
34 //    +--------------+---------+---------+-------+-------------+-------------------+
35 //    |   Fixed32    | Fixed32 | Fixed32 | char  |    char     | Fixed64   Fixed64 |
36 //    +--------------+---------+---------+-------+-------------+-------------------+
37 //
38 // List of flags:
39 //   has_ttl: Whether the file contain TTL data.
40 //
41 // Expiration range in the header is a rough range based on
42 // blob_db_options.ttl_range_secs.
43 struct BlobLogHeader {
44   static constexpr size_t kSize = 30;
45 
46   BlobLogHeader() = default;
BlobLogHeaderBlobLogHeader47   BlobLogHeader(uint32_t _column_family_id, CompressionType _compression,
48                 bool _has_ttl, const ExpirationRange& _expiration_range)
49       : column_family_id(_column_family_id),
50         compression(_compression),
51         has_ttl(_has_ttl),
52         expiration_range(_expiration_range) {}
53 
54   uint32_t version = kVersion1;
55   uint32_t column_family_id = 0;
56   CompressionType compression = kNoCompression;
57   bool has_ttl = false;
58   ExpirationRange expiration_range;
59 
60   void EncodeTo(std::string* dst);
61 
62   Status DecodeFrom(Slice slice);
63 };
64 
65 // Format of blob log file footer (32 bytes):
66 //
67 //    +--------------+------------+-------------------+------------+
68 //    | magic number | blob count | expiration range  | footer CRC |
69 //    +--------------+------------+-------------------+------------+
70 //    |   Fixed32    |  Fixed64   | Fixed64 + Fixed64 |   Fixed32  |
71 //    +--------------+------------+-------------------+------------+
72 //
73 // The footer will be presented only when the blob file is properly closed.
74 //
75 // Unlike the same field in file header, expiration range in the footer is the
76 // range of smallest and largest expiration of the data in this file.
77 struct BlobLogFooter {
78   static constexpr size_t kSize = 32;
79 
80   uint64_t blob_count = 0;
81   ExpirationRange expiration_range = std::make_pair(0, 0);
82   uint32_t crc = 0;
83 
84   void EncodeTo(std::string* dst);
85 
86   Status DecodeFrom(Slice slice);
87 };
88 
89 // Blob record format (32 bytes header + key + value):
90 //
91 //    +------------+--------------+------------+------------+----------+---------+-----------+
92 //    | key length | value length | expiration | header CRC | blob CRC |   key   |   value   |
93 //    +------------+--------------+------------+------------+----------+---------+-----------+
94 //    |   Fixed64  |   Fixed64    |  Fixed64   |  Fixed32   | Fixed32  | key len | value len |
95 //    +------------+--------------+------------+------------+----------+---------+-----------+
96 //
97 // If file has has_ttl = false, expiration field is always 0, and the blob
98 // doesn't has expiration.
99 //
100 // Also note that if compression is used, value is compressed value and value
101 // length is compressed value length.
102 //
103 // Header CRC is the checksum of (key_len + val_len + expiration), while
104 // blob CRC is the checksum of (key + value).
105 //
106 // We could use variable length encoding (Varint64) to save more space, but it
107 // make reader more complicated.
108 struct BlobLogRecord {
109   // header include fields up to blob CRC
110   static constexpr size_t kHeaderSize = 32;
111 
112   uint64_t key_size = 0;
113   uint64_t value_size = 0;
114   uint64_t expiration = 0;
115   uint32_t header_crc = 0;
116   uint32_t blob_crc = 0;
117   Slice key;
118   Slice value;
119   std::unique_ptr<char[]> key_buf;
120   std::unique_ptr<char[]> value_buf;
121 
record_sizeBlobLogRecord122   uint64_t record_size() const { return kHeaderSize + key_size + value_size; }
123 
124   void EncodeHeaderTo(std::string* dst);
125 
126   Status DecodeHeaderFrom(Slice src);
127 
128   Status CheckBlobCRC() const;
129 };
130 
131 }  // namespace blob_db
132 }  // namespace ROCKSDB_NAMESPACE
133 #endif  // ROCKSDB_LITE
134