1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011-2019 Pawel Jakub Dawidek <[email protected]>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 #include <sys/param.h>
31 #ifdef _KERNEL
32 #include <sys/kernel.h>
33 #include <sys/malloc.h>
34 #include <sys/sysctl.h>
35 #include <sys/systm.h>
36 #endif /* _KERNEL */
37 #include <sys/queue.h>
38 #include <sys/tree.h>
39
40 #include <geom/geom.h>
41
42 #include <geom/eli/g_eli.h>
43
44 #ifdef _KERNEL
45 MALLOC_DECLARE(M_ELI);
46
47 SYSCTL_DECL(_kern_geom_eli);
48 /*
49 * The default limit (8192 keys) will allow to cache all keys for 4TB
50 * provider with 512 bytes sectors and will take around 1MB of memory.
51 */
52 static u_int g_eli_key_cache_limit = 8192;
53 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, key_cache_limit, CTLFLAG_RDTUN,
54 &g_eli_key_cache_limit, 0, "Maximum number of encryption keys to cache");
55 static uint64_t g_eli_key_cache_hits;
56 SYSCTL_UQUAD(_kern_geom_eli, OID_AUTO, key_cache_hits, CTLFLAG_RW,
57 &g_eli_key_cache_hits, 0, "Key cache hits");
58 static uint64_t g_eli_key_cache_misses;
59 SYSCTL_UQUAD(_kern_geom_eli, OID_AUTO, key_cache_misses, CTLFLAG_RW,
60 &g_eli_key_cache_misses, 0, "Key cache misses");
61
62 static int
g_eli_key_cmp(const struct g_eli_key * a,const struct g_eli_key * b)63 g_eli_key_cmp(const struct g_eli_key *a, const struct g_eli_key *b)
64 {
65
66 if (a->gek_keyno > b->gek_keyno)
67 return (1);
68 else if (a->gek_keyno < b->gek_keyno)
69 return (-1);
70 return (0);
71 }
72 #endif /* _KERNEL */
73
74 void
g_eli_key_fill(struct g_eli_softc * sc,struct g_eli_key * key,uint64_t keyno)75 g_eli_key_fill(struct g_eli_softc *sc, struct g_eli_key *key, uint64_t keyno)
76 {
77 const uint8_t *ekey;
78 struct {
79 char magic[4];
80 uint8_t keyno[8];
81 } __packed hmacdata;
82
83 if ((sc->sc_flags & G_ELI_FLAG_ENC_IVKEY) != 0)
84 ekey = sc->sc_mkey;
85 else
86 ekey = sc->sc_ekey;
87
88 bcopy("ekey", hmacdata.magic, 4);
89 le64enc(hmacdata.keyno, keyno);
90 g_eli_crypto_hmac(ekey, G_ELI_MAXKEYLEN, (uint8_t *)&hmacdata,
91 sizeof(hmacdata), key->gek_key, 0);
92 key->gek_keyno = keyno;
93 key->gek_count = 0;
94 key->gek_magic = G_ELI_KEY_MAGIC;
95 }
96
97 #ifdef _KERNEL
98 RB_PROTOTYPE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp);
99 RB_GENERATE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp);
100
101 static struct g_eli_key *
g_eli_key_allocate(struct g_eli_softc * sc,uint64_t keyno)102 g_eli_key_allocate(struct g_eli_softc *sc, uint64_t keyno)
103 {
104 struct g_eli_key *key, *ekey, keysearch;
105
106 mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
107 mtx_unlock(&sc->sc_ekeys_lock);
108
109 key = malloc(sizeof(*key), M_ELI, M_WAITOK);
110 g_eli_key_fill(sc, key, keyno);
111
112 mtx_lock(&sc->sc_ekeys_lock);
113 /*
114 * Recheck if the key wasn't added while we weren't holding the lock.
115 */
116 keysearch.gek_keyno = keyno;
117 ekey = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch);
118 if (ekey != NULL) {
119 zfree(key, M_ELI);
120 key = ekey;
121 TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
122 } else {
123 RB_INSERT(g_eli_key_tree, &sc->sc_ekeys_tree, key);
124 sc->sc_ekeys_allocated++;
125 }
126 TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next);
127
128 return (key);
129 }
130
131 static struct g_eli_key *
g_eli_key_find_last(struct g_eli_softc * sc)132 g_eli_key_find_last(struct g_eli_softc *sc)
133 {
134 struct g_eli_key *key;
135
136 mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
137
138 TAILQ_FOREACH(key, &sc->sc_ekeys_queue, gek_next) {
139 if (key->gek_count == 0)
140 break;
141 }
142
143 return (key);
144 }
145
146 static void
g_eli_key_replace(struct g_eli_softc * sc,struct g_eli_key * key,uint64_t keyno)147 g_eli_key_replace(struct g_eli_softc *sc, struct g_eli_key *key, uint64_t keyno)
148 {
149
150 mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
151 KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid magic."));
152
153 RB_REMOVE(g_eli_key_tree, &sc->sc_ekeys_tree, key);
154 TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
155
156 KASSERT(key->gek_count == 0, ("gek_count=%d", key->gek_count));
157
158 g_eli_key_fill(sc, key, keyno);
159
160 RB_INSERT(g_eli_key_tree, &sc->sc_ekeys_tree, key);
161 TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next);
162 }
163
164 static void
g_eli_key_remove(struct g_eli_softc * sc,struct g_eli_key * key)165 g_eli_key_remove(struct g_eli_softc *sc, struct g_eli_key *key)
166 {
167
168 mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
169 KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid magic."));
170 KASSERT(key->gek_count == 0, ("gek_count=%d", key->gek_count));
171
172 RB_REMOVE(g_eli_key_tree, &sc->sc_ekeys_tree, key);
173 TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
174 sc->sc_ekeys_allocated--;
175 zfree(key, M_ELI);
176 }
177
178 void
g_eli_key_init(struct g_eli_softc * sc)179 g_eli_key_init(struct g_eli_softc *sc)
180 {
181 uint8_t *mkey;
182
183 mtx_lock(&sc->sc_ekeys_lock);
184
185 mkey = sc->sc_mkey + sizeof(sc->sc_ivkey);
186 if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0)
187 bcopy(mkey, sc->sc_ekey, G_ELI_DATAKEYLEN);
188 else {
189 /*
190 * The encryption key is: ekey = HMAC_SHA512(Data-Key, 0x10)
191 */
192 g_eli_crypto_hmac(mkey, G_ELI_MAXKEYLEN, "\x10", 1,
193 sc->sc_ekey, 0);
194 }
195
196 if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) {
197 sc->sc_ekeys_total = 1;
198 sc->sc_ekeys_allocated = 0;
199 } else {
200 off_t mediasize;
201 size_t blocksize;
202
203 if ((sc->sc_flags & G_ELI_FLAG_AUTH) != 0) {
204 struct g_provider *pp;
205
206 pp = LIST_FIRST(&sc->sc_geom->consumer)->provider;
207 mediasize = pp->mediasize;
208 blocksize = pp->sectorsize;
209 } else {
210 mediasize = sc->sc_mediasize;
211 blocksize = sc->sc_sectorsize;
212 }
213 sc->sc_ekeys_total =
214 ((mediasize - 1) >> G_ELI_KEY_SHIFT) / blocksize + 1;
215 sc->sc_ekeys_allocated = 0;
216 TAILQ_INIT(&sc->sc_ekeys_queue);
217 RB_INIT(&sc->sc_ekeys_tree);
218 if (sc->sc_ekeys_total <= g_eli_key_cache_limit) {
219 uint64_t keyno;
220
221 for (keyno = 0; keyno < sc->sc_ekeys_total; keyno++)
222 (void)g_eli_key_allocate(sc, keyno);
223 KASSERT(sc->sc_ekeys_total == sc->sc_ekeys_allocated,
224 ("sc_ekeys_total=%ju != sc_ekeys_allocated=%ju",
225 (uintmax_t)sc->sc_ekeys_total,
226 (uintmax_t)sc->sc_ekeys_allocated));
227 }
228 }
229
230 mtx_unlock(&sc->sc_ekeys_lock);
231 }
232
233 void
g_eli_key_destroy(struct g_eli_softc * sc)234 g_eli_key_destroy(struct g_eli_softc *sc)
235 {
236
237 mtx_lock(&sc->sc_ekeys_lock);
238 if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) {
239 explicit_bzero(sc->sc_ekey, sizeof(sc->sc_ekey));
240 } else {
241 struct g_eli_key *key;
242
243 while ((key = TAILQ_FIRST(&sc->sc_ekeys_queue)) != NULL)
244 g_eli_key_remove(sc, key);
245 TAILQ_INIT(&sc->sc_ekeys_queue);
246 RB_INIT(&sc->sc_ekeys_tree);
247 }
248 mtx_unlock(&sc->sc_ekeys_lock);
249 }
250
251 void
g_eli_key_resize(struct g_eli_softc * sc)252 g_eli_key_resize(struct g_eli_softc *sc)
253 {
254 uint64_t new_ekeys_total;
255 off_t mediasize;
256 size_t blocksize;
257
258 if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) {
259 return;
260 }
261
262 mtx_lock(&sc->sc_ekeys_lock);
263
264 if ((sc->sc_flags & G_ELI_FLAG_AUTH) != 0) {
265 struct g_provider *pp;
266
267 pp = LIST_FIRST(&sc->sc_geom->consumer)->provider;
268 mediasize = pp->mediasize;
269 blocksize = pp->sectorsize;
270 } else {
271 mediasize = sc->sc_mediasize;
272 blocksize = sc->sc_sectorsize;
273 }
274 new_ekeys_total = ((mediasize - 1) >> G_ELI_KEY_SHIFT) / blocksize + 1;
275 /* We only allow to grow. */
276 KASSERT(new_ekeys_total >= sc->sc_ekeys_total,
277 ("new_ekeys_total=%ju < sc_ekeys_total=%ju",
278 (uintmax_t)new_ekeys_total, (uintmax_t)sc->sc_ekeys_total));
279 if (new_ekeys_total <= g_eli_key_cache_limit) {
280 uint64_t keyno;
281
282 for (keyno = sc->sc_ekeys_total; keyno < new_ekeys_total;
283 keyno++) {
284 (void)g_eli_key_allocate(sc, keyno);
285 }
286 KASSERT(new_ekeys_total == sc->sc_ekeys_allocated,
287 ("new_ekeys_total=%ju != sc_ekeys_allocated=%ju",
288 (uintmax_t)new_ekeys_total,
289 (uintmax_t)sc->sc_ekeys_allocated));
290 }
291
292 sc->sc_ekeys_total = new_ekeys_total;
293
294 mtx_unlock(&sc->sc_ekeys_lock);
295 }
296
297 /*
298 * Select encryption key. If G_ELI_FLAG_SINGLE_KEY is present we only have one
299 * key available for all the data. If the flag is not present select the key
300 * based on data offset.
301 */
302 uint8_t *
g_eli_key_hold(struct g_eli_softc * sc,off_t offset,size_t blocksize)303 g_eli_key_hold(struct g_eli_softc *sc, off_t offset, size_t blocksize)
304 {
305 struct g_eli_key *key, keysearch;
306 uint64_t keyno;
307
308 if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0)
309 return (sc->sc_ekey);
310
311 /* We switch key every 2^G_ELI_KEY_SHIFT blocks. */
312 keyno = (offset >> G_ELI_KEY_SHIFT) / blocksize;
313
314 KASSERT(keyno < sc->sc_ekeys_total,
315 ("%s: keyno=%ju >= sc_ekeys_total=%ju",
316 __func__, (uintmax_t)keyno, (uintmax_t)sc->sc_ekeys_total));
317
318 keysearch.gek_keyno = keyno;
319
320 if (sc->sc_ekeys_total == sc->sc_ekeys_allocated) {
321 /* We have all the keys, so avoid some overhead. */
322 key = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch);
323 KASSERT(key != NULL, ("No key %ju found.", (uintmax_t)keyno));
324 KASSERT(key->gek_magic == G_ELI_KEY_MAGIC,
325 ("Invalid key magic."));
326 return (key->gek_key);
327 }
328
329 mtx_lock(&sc->sc_ekeys_lock);
330 key = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch);
331 if (key != NULL) {
332 g_eli_key_cache_hits++;
333 TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
334 TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next);
335 } else {
336 /*
337 * No key in cache, find the least recently unreferenced key
338 * or allocate one if we haven't reached our limit yet.
339 */
340 if (sc->sc_ekeys_allocated < g_eli_key_cache_limit) {
341 key = g_eli_key_allocate(sc, keyno);
342 } else {
343 g_eli_key_cache_misses++;
344 key = g_eli_key_find_last(sc);
345 if (key != NULL) {
346 g_eli_key_replace(sc, key, keyno);
347 } else {
348 /* All keys are referenced? Allocate one. */
349 key = g_eli_key_allocate(sc, keyno);
350 }
351 }
352 }
353 key->gek_count++;
354 mtx_unlock(&sc->sc_ekeys_lock);
355
356 KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid key magic."));
357
358 return (key->gek_key);
359 }
360
361 void
g_eli_key_drop(struct g_eli_softc * sc,uint8_t * rawkey)362 g_eli_key_drop(struct g_eli_softc *sc, uint8_t *rawkey)
363 {
364 struct g_eli_key *key = (struct g_eli_key *)rawkey;
365
366 if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0)
367 return;
368
369 KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid key magic."));
370
371 if (sc->sc_ekeys_total == sc->sc_ekeys_allocated)
372 return;
373
374 mtx_lock(&sc->sc_ekeys_lock);
375 KASSERT(key->gek_count > 0, ("key->gek_count=%d", key->gek_count));
376 key->gek_count--;
377 while (sc->sc_ekeys_allocated > g_eli_key_cache_limit) {
378 key = g_eli_key_find_last(sc);
379 if (key == NULL)
380 break;
381 g_eli_key_remove(sc, key);
382 }
383 mtx_unlock(&sc->sc_ekeys_lock);
384 }
385 #endif /* _KERNEL */
386