1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include <sys/zfs_context.h>
26 #include <sys/crypto/icp.h>
27 #include <sys/crypto/spi.h>
28 #include <sys/simd.h>
29 #include <modes/modes.h>
30 #include <aes/aes_impl.h>
31
32 /*
33 * Initialize AES encryption and decryption key schedules.
34 *
35 * Parameters:
36 * cipherKey User key
37 * keyBits AES key size (128, 192, or 256 bits)
38 * keysched AES key schedule to be initialized, of type aes_key_t.
39 * Allocated by aes_alloc_keysched().
40 */
41 void
aes_init_keysched(const uint8_t * cipherKey,uint_t keyBits,void * keysched)42 aes_init_keysched(const uint8_t *cipherKey, uint_t keyBits, void *keysched)
43 {
44 const aes_impl_ops_t *ops = aes_impl_get_ops();
45 aes_key_t *newbie = keysched;
46 uint_t keysize, i, j;
47 union {
48 uint64_t ka64[4];
49 uint32_t ka32[8];
50 } keyarr;
51
52 switch (keyBits) {
53 case 128:
54 newbie->nr = 10;
55 break;
56
57 case 192:
58 newbie->nr = 12;
59 break;
60
61 case 256:
62 newbie->nr = 14;
63 break;
64
65 default:
66 /* should never get here */
67 return;
68 }
69 keysize = CRYPTO_BITS2BYTES(keyBits);
70
71 /*
72 * Generic C implementation requires byteswap for little endian
73 * machines, various accelerated implementations for various
74 * architectures may not.
75 */
76 if (!ops->needs_byteswap) {
77 /* no byteswap needed */
78 if (IS_P2ALIGNED(cipherKey, sizeof (uint64_t))) {
79 for (i = 0, j = 0; j < keysize; i++, j += 8) {
80 /* LINTED: pointer alignment */
81 keyarr.ka64[i] = *((uint64_t *)&cipherKey[j]);
82 }
83 } else {
84 bcopy(cipherKey, keyarr.ka32, keysize);
85 }
86 } else {
87 /* byte swap */
88 for (i = 0, j = 0; j < keysize; i++, j += 4) {
89 keyarr.ka32[i] =
90 htonl(*(uint32_t *)(void *)&cipherKey[j]);
91 }
92 }
93
94 ops->generate(newbie, keyarr.ka32, keyBits);
95 newbie->ops = ops;
96
97 /*
98 * Note: if there are systems that need the AES_64BIT_KS type in the
99 * future, move setting key schedule type to individual implementations
100 */
101 newbie->type = AES_32BIT_KS;
102 }
103
104
105 /*
106 * Encrypt one block using AES.
107 * Align if needed and (for x86 32-bit only) byte-swap.
108 *
109 * Parameters:
110 * ks Key schedule, of type aes_key_t
111 * pt Input block (plain text)
112 * ct Output block (crypto text). Can overlap with pt
113 */
114 int
aes_encrypt_block(const void * ks,const uint8_t * pt,uint8_t * ct)115 aes_encrypt_block(const void *ks, const uint8_t *pt, uint8_t *ct)
116 {
117 aes_key_t *ksch = (aes_key_t *)ks;
118 const aes_impl_ops_t *ops = ksch->ops;
119
120 if (IS_P2ALIGNED2(pt, ct, sizeof (uint32_t)) && !ops->needs_byteswap) {
121 /* LINTED: pointer alignment */
122 ops->encrypt(&ksch->encr_ks.ks32[0], ksch->nr,
123 /* LINTED: pointer alignment */
124 (uint32_t *)pt, (uint32_t *)ct);
125 } else {
126 uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)];
127
128 /* Copy input block into buffer */
129 if (ops->needs_byteswap) {
130 buffer[0] = htonl(*(uint32_t *)(void *)&pt[0]);
131 buffer[1] = htonl(*(uint32_t *)(void *)&pt[4]);
132 buffer[2] = htonl(*(uint32_t *)(void *)&pt[8]);
133 buffer[3] = htonl(*(uint32_t *)(void *)&pt[12]);
134 } else
135 bcopy(pt, &buffer, AES_BLOCK_LEN);
136
137 ops->encrypt(&ksch->encr_ks.ks32[0], ksch->nr, buffer, buffer);
138
139 /* Copy result from buffer to output block */
140 if (ops->needs_byteswap) {
141 *(uint32_t *)(void *)&ct[0] = htonl(buffer[0]);
142 *(uint32_t *)(void *)&ct[4] = htonl(buffer[1]);
143 *(uint32_t *)(void *)&ct[8] = htonl(buffer[2]);
144 *(uint32_t *)(void *)&ct[12] = htonl(buffer[3]);
145 } else
146 bcopy(&buffer, ct, AES_BLOCK_LEN);
147 }
148 return (CRYPTO_SUCCESS);
149 }
150
151
152 /*
153 * Decrypt one block using AES.
154 * Align and byte-swap if needed.
155 *
156 * Parameters:
157 * ks Key schedule, of type aes_key_t
158 * ct Input block (crypto text)
159 * pt Output block (plain text). Can overlap with pt
160 */
161 int
aes_decrypt_block(const void * ks,const uint8_t * ct,uint8_t * pt)162 aes_decrypt_block(const void *ks, const uint8_t *ct, uint8_t *pt)
163 {
164 aes_key_t *ksch = (aes_key_t *)ks;
165 const aes_impl_ops_t *ops = ksch->ops;
166
167 if (IS_P2ALIGNED2(ct, pt, sizeof (uint32_t)) && !ops->needs_byteswap) {
168 /* LINTED: pointer alignment */
169 ops->decrypt(&ksch->decr_ks.ks32[0], ksch->nr,
170 /* LINTED: pointer alignment */
171 (uint32_t *)ct, (uint32_t *)pt);
172 } else {
173 uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)];
174
175 /* Copy input block into buffer */
176 if (ops->needs_byteswap) {
177 buffer[0] = htonl(*(uint32_t *)(void *)&ct[0]);
178 buffer[1] = htonl(*(uint32_t *)(void *)&ct[4]);
179 buffer[2] = htonl(*(uint32_t *)(void *)&ct[8]);
180 buffer[3] = htonl(*(uint32_t *)(void *)&ct[12]);
181 } else
182 bcopy(ct, &buffer, AES_BLOCK_LEN);
183
184 ops->decrypt(&ksch->decr_ks.ks32[0], ksch->nr, buffer, buffer);
185
186 /* Copy result from buffer to output block */
187 if (ops->needs_byteswap) {
188 *(uint32_t *)(void *)&pt[0] = htonl(buffer[0]);
189 *(uint32_t *)(void *)&pt[4] = htonl(buffer[1]);
190 *(uint32_t *)(void *)&pt[8] = htonl(buffer[2]);
191 *(uint32_t *)(void *)&pt[12] = htonl(buffer[3]);
192 } else
193 bcopy(&buffer, pt, AES_BLOCK_LEN);
194 }
195 return (CRYPTO_SUCCESS);
196 }
197
198
199 /*
200 * Allocate key schedule for AES.
201 *
202 * Return the pointer and set size to the number of bytes allocated.
203 * Memory allocated must be freed by the caller when done.
204 *
205 * Parameters:
206 * size Size of key schedule allocated, in bytes
207 * kmflag Flag passed to kmem_alloc(9F); ignored in userland.
208 */
209 /* ARGSUSED */
210 void *
aes_alloc_keysched(size_t * size,int kmflag)211 aes_alloc_keysched(size_t *size, int kmflag)
212 {
213 aes_key_t *keysched;
214
215 keysched = (aes_key_t *)kmem_alloc(sizeof (aes_key_t), kmflag);
216 if (keysched != NULL) {
217 *size = sizeof (aes_key_t);
218 return (keysched);
219 }
220 return (NULL);
221 }
222
223 /* AES implementation that contains the fastest methods */
224 static aes_impl_ops_t aes_fastest_impl = {
225 .name = "fastest"
226 };
227
228 /* All compiled in implementations */
229 const aes_impl_ops_t *aes_all_impl[] = {
230 &aes_generic_impl,
231 #if defined(__x86_64)
232 &aes_x86_64_impl,
233 #endif
234 #if defined(__x86_64) && defined(HAVE_AES)
235 &aes_aesni_impl,
236 #endif
237 };
238
239 /* Indicate that benchmark has been completed */
240 static boolean_t aes_impl_initialized = B_FALSE;
241
242 /* Select aes implementation */
243 #define IMPL_FASTEST (UINT32_MAX)
244 #define IMPL_CYCLE (UINT32_MAX-1)
245
246 #define AES_IMPL_READ(i) (*(volatile uint32_t *) &(i))
247
248 static uint32_t icp_aes_impl = IMPL_FASTEST;
249 static uint32_t user_sel_impl = IMPL_FASTEST;
250
251 /* Hold all supported implementations */
252 static size_t aes_supp_impl_cnt = 0;
253 static aes_impl_ops_t *aes_supp_impl[ARRAY_SIZE(aes_all_impl)];
254
255 /*
256 * Returns the AES operations for encrypt/decrypt/key setup. When a
257 * SIMD implementation is not allowed in the current context, then
258 * fallback to the fastest generic implementation.
259 */
260 const aes_impl_ops_t *
aes_impl_get_ops(void)261 aes_impl_get_ops(void)
262 {
263 if (!kfpu_allowed())
264 return (&aes_generic_impl);
265
266 const aes_impl_ops_t *ops = NULL;
267 const uint32_t impl = AES_IMPL_READ(icp_aes_impl);
268
269 switch (impl) {
270 case IMPL_FASTEST:
271 ASSERT(aes_impl_initialized);
272 ops = &aes_fastest_impl;
273 break;
274 case IMPL_CYCLE:
275 /* Cycle through supported implementations */
276 ASSERT(aes_impl_initialized);
277 ASSERT3U(aes_supp_impl_cnt, >, 0);
278 static size_t cycle_impl_idx = 0;
279 size_t idx = (++cycle_impl_idx) % aes_supp_impl_cnt;
280 ops = aes_supp_impl[idx];
281 break;
282 default:
283 ASSERT3U(impl, <, aes_supp_impl_cnt);
284 ASSERT3U(aes_supp_impl_cnt, >, 0);
285 if (impl < ARRAY_SIZE(aes_all_impl))
286 ops = aes_supp_impl[impl];
287 break;
288 }
289
290 ASSERT3P(ops, !=, NULL);
291
292 return (ops);
293 }
294
295 /*
296 * Initialize all supported implementations.
297 */
298 void
aes_impl_init(void)299 aes_impl_init(void)
300 {
301 aes_impl_ops_t *curr_impl;
302 int i, c;
303
304 /* Move supported implementations into aes_supp_impls */
305 for (i = 0, c = 0; i < ARRAY_SIZE(aes_all_impl); i++) {
306 curr_impl = (aes_impl_ops_t *)aes_all_impl[i];
307
308 if (curr_impl->is_supported())
309 aes_supp_impl[c++] = (aes_impl_ops_t *)curr_impl;
310 }
311 aes_supp_impl_cnt = c;
312
313 /*
314 * Set the fastest implementation given the assumption that the
315 * hardware accelerated version is the fastest.
316 */
317 #if defined(__x86_64)
318 #if defined(HAVE_AES)
319 if (aes_aesni_impl.is_supported()) {
320 memcpy(&aes_fastest_impl, &aes_aesni_impl,
321 sizeof (aes_fastest_impl));
322 } else
323 #endif
324 {
325 memcpy(&aes_fastest_impl, &aes_x86_64_impl,
326 sizeof (aes_fastest_impl));
327 }
328 #else
329 memcpy(&aes_fastest_impl, &aes_generic_impl,
330 sizeof (aes_fastest_impl));
331 #endif
332
333 strlcpy(aes_fastest_impl.name, "fastest", AES_IMPL_NAME_MAX);
334
335 /* Finish initialization */
336 atomic_swap_32(&icp_aes_impl, user_sel_impl);
337 aes_impl_initialized = B_TRUE;
338 }
339
340 static const struct {
341 char *name;
342 uint32_t sel;
343 } aes_impl_opts[] = {
344 { "cycle", IMPL_CYCLE },
345 { "fastest", IMPL_FASTEST },
346 };
347
348 /*
349 * Function sets desired aes implementation.
350 *
351 * If we are called before init(), user preference will be saved in
352 * user_sel_impl, and applied in later init() call. This occurs when module
353 * parameter is specified on module load. Otherwise, directly update
354 * icp_aes_impl.
355 *
356 * @val Name of aes implementation to use
357 * @param Unused.
358 */
359 int
aes_impl_set(const char * val)360 aes_impl_set(const char *val)
361 {
362 int err = -EINVAL;
363 char req_name[AES_IMPL_NAME_MAX];
364 uint32_t impl = AES_IMPL_READ(user_sel_impl);
365 size_t i;
366
367 /* sanitize input */
368 i = strnlen(val, AES_IMPL_NAME_MAX);
369 if (i == 0 || i >= AES_IMPL_NAME_MAX)
370 return (err);
371
372 strlcpy(req_name, val, AES_IMPL_NAME_MAX);
373 while (i > 0 && isspace(req_name[i-1]))
374 i--;
375 req_name[i] = '\0';
376
377 /* Check mandatory options */
378 for (i = 0; i < ARRAY_SIZE(aes_impl_opts); i++) {
379 if (strcmp(req_name, aes_impl_opts[i].name) == 0) {
380 impl = aes_impl_opts[i].sel;
381 err = 0;
382 break;
383 }
384 }
385
386 /* check all supported impl if init() was already called */
387 if (err != 0 && aes_impl_initialized) {
388 /* check all supported implementations */
389 for (i = 0; i < aes_supp_impl_cnt; i++) {
390 if (strcmp(req_name, aes_supp_impl[i]->name) == 0) {
391 impl = i;
392 err = 0;
393 break;
394 }
395 }
396 }
397
398 if (err == 0) {
399 if (aes_impl_initialized)
400 atomic_swap_32(&icp_aes_impl, impl);
401 else
402 atomic_swap_32(&user_sel_impl, impl);
403 }
404
405 return (err);
406 }
407
408 #if defined(_KERNEL) && defined(__linux__)
409
410 static int
icp_aes_impl_set(const char * val,zfs_kernel_param_t * kp)411 icp_aes_impl_set(const char *val, zfs_kernel_param_t *kp)
412 {
413 return (aes_impl_set(val));
414 }
415
416 static int
icp_aes_impl_get(char * buffer,zfs_kernel_param_t * kp)417 icp_aes_impl_get(char *buffer, zfs_kernel_param_t *kp)
418 {
419 int i, cnt = 0;
420 char *fmt;
421 const uint32_t impl = AES_IMPL_READ(icp_aes_impl);
422
423 ASSERT(aes_impl_initialized);
424
425 /* list mandatory options */
426 for (i = 0; i < ARRAY_SIZE(aes_impl_opts); i++) {
427 fmt = (impl == aes_impl_opts[i].sel) ? "[%s] " : "%s ";
428 cnt += sprintf(buffer + cnt, fmt, aes_impl_opts[i].name);
429 }
430
431 /* list all supported implementations */
432 for (i = 0; i < aes_supp_impl_cnt; i++) {
433 fmt = (i == impl) ? "[%s] " : "%s ";
434 cnt += sprintf(buffer + cnt, fmt, aes_supp_impl[i]->name);
435 }
436
437 return (cnt);
438 }
439
440 module_param_call(icp_aes_impl, icp_aes_impl_set, icp_aes_impl_get,
441 NULL, 0644);
442 MODULE_PARM_DESC(icp_aes_impl, "Select aes implementation.");
443 #endif
444