1 /*-
2 * Copyright (c) 2017 Ruslan Bukin <[email protected]>
3 * All rights reserved.
4 *
5 * This software was developed by BAE Systems, the University of Cambridge
6 * Computer Laboratory, and Memorial University under DARPA/AFRL contract
7 * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing
8 * (TC) research program.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 /*
33 * Design overview.
34 *
35 * The driver provides character device for mmap(2) and ioctl(2) system calls
36 * allowing user to manage isolated compartments ("enclaves") in user VA space.
37 *
38 * The driver duties is EPC pages management, enclave management, user data
39 * validation.
40 *
41 * This driver requires Intel SGX support from hardware.
42 *
43 * /dev/sgx:
44 * .mmap:
45 * sgx_mmap_single() allocates VM object with following pager
46 * operations:
47 * a) sgx_pg_ctor():
48 * VM object constructor does nothing
49 * b) sgx_pg_dtor():
50 * VM object destructor destroys the SGX enclave associated
51 * with the object: it frees all the EPC pages allocated for
52 * enclave and removes the enclave.
53 * c) sgx_pg_fault():
54 * VM object fault handler does nothing
55 *
56 * .ioctl:
57 * sgx_ioctl():
58 * a) SGX_IOC_ENCLAVE_CREATE
59 * Adds Enclave SECS page: initial step of enclave creation.
60 * b) SGX_IOC_ENCLAVE_ADD_PAGE
61 * Adds TCS, REG pages to the enclave.
62 * c) SGX_IOC_ENCLAVE_INIT
63 * Finalizes enclave creation.
64 *
65 * Enclave lifecycle:
66 * .-- ECREATE -- Add SECS page
67 * Kernel | EADD -- Add TCS, REG pages
68 * space | EEXTEND -- Measure the page (take unique hash)
69 * ENCLS | EPA -- Allocate version array page
70 * '-- EINIT -- Finalize enclave creation
71 * User .-- EENTER -- Go to entry point of enclave
72 * space | EEXIT -- Exit back to main application
73 * ENCLU '-- ERESUME -- Resume enclave execution (e.g. after exception)
74 *
75 * Enclave lifecycle from driver point of view:
76 * 1) User calls mmap() on /dev/sgx: we allocate a VM object
77 * 2) User calls ioctl SGX_IOC_ENCLAVE_CREATE: we look for the VM object
78 * associated with user process created on step 1, create SECS physical
79 * page and store it in enclave's VM object queue by special index
80 * SGX_SECS_VM_OBJECT_INDEX.
81 * 3) User calls ioctl SGX_IOC_ENCLAVE_ADD_PAGE: we look for enclave created
82 * on step 2, create TCS or REG physical page and map it to specified by
83 * user address of enclave VM object.
84 * 4) User finalizes enclave creation with ioctl SGX_IOC_ENCLAVE_INIT call.
85 * 5) User can freely enter to and exit from enclave using ENCLU instructions
86 * from userspace: the driver does nothing here.
87 * 6) User proceed munmap(2) system call (or the process with enclave dies):
88 * we destroy the enclave associated with the object.
89 *
90 * EPC page types and their indexes in VM object queue:
91 * - PT_SECS index is special and equals SGX_SECS_VM_OBJECT_INDEX (-1);
92 * - PT_TCS and PT_REG indexes are specified by user in addr field of ioctl
93 * request data and determined as follows:
94 * pidx = OFF_TO_IDX(addp->addr - vmh->base);
95 * - PT_VA index is special, created for PT_REG, PT_TCS and PT_SECS pages
96 * and determined by formula:
97 * va_page_idx = - SGX_VA_PAGES_OFFS - (page_idx / SGX_VA_PAGE_SLOTS);
98 * PT_VA page can hold versions of up to 512 pages, and slot for each
99 * page in PT_VA page is determined as follows:
100 * va_slot_idx = page_idx % SGX_VA_PAGE_SLOTS;
101 * - PT_TRIM is unused.
102 *
103 * Locking:
104 * SGX ENCLS set of instructions have limitations on concurrency:
105 * some instructions can't be executed same time on different CPUs.
106 * We use sc->mtx_encls lock around them to prevent concurrent execution.
107 * sc->mtx lock is used to manage list of created enclaves and the state of
108 * SGX driver.
109 *
110 * Eviction of EPC pages:
111 * Eviction support is not implemented in this driver, however the driver
112 * manages VA (version array) pages: it allocates a VA slot for each EPC
113 * page. This will be required for eviction support in future.
114 * VA pages and slots are currently unused.
115 *
116 * Intel® 64 and IA-32 Architectures Software Developer's Manual
117 * https://software.intel.com/en-us/articles/intel-sdm
118 */
119
120 #include <sys/cdefs.h>
121 #include <sys/param.h>
122 #include <sys/systm.h>
123 #include <sys/ioccom.h>
124 #include <sys/malloc.h>
125 #include <sys/kernel.h>
126 #include <sys/lock.h>
127 #include <sys/mutex.h>
128 #include <sys/rwlock.h>
129 #include <sys/conf.h>
130 #include <sys/module.h>
131 #include <sys/proc.h>
132 #include <sys/vmem.h>
133 #include <sys/vmmeter.h>
134
135 #include <vm/vm.h>
136 #include <vm/vm_param.h>
137 #include <vm/vm_extern.h>
138 #include <vm/vm_kern.h>
139 #include <vm/vm_page.h>
140 #include <vm/vm_map.h>
141 #include <vm/vm_object.h>
142 #include <vm/vm_pager.h>
143 #include <vm/vm_phys.h>
144 #include <vm/vm_radix.h>
145 #include <vm/pmap.h>
146
147 #include <machine/md_var.h>
148 #include <machine/specialreg.h>
149 #include <machine/cpufunc.h>
150 #include <machine/sgx.h>
151 #include <machine/sgxreg.h>
152
153 #include <amd64/sgx/sgxvar.h>
154
155 #define SGX_DEBUG
156 #undef SGX_DEBUG
157
158 #ifdef SGX_DEBUG
159 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
160 #else
161 #define dprintf(fmt, ...)
162 #endif
163
164 static struct cdev_pager_ops sgx_pg_ops;
165 struct sgx_softc sgx_sc;
166
167 static int
sgx_get_epc_page(struct sgx_softc * sc,struct epc_page ** epc)168 sgx_get_epc_page(struct sgx_softc *sc, struct epc_page **epc)
169 {
170 vmem_addr_t addr;
171 int i;
172
173 if (vmem_alloc(sc->vmem_epc, PAGE_SIZE, M_FIRSTFIT | M_NOWAIT,
174 &addr) == 0) {
175 i = (addr - sc->epc_base) / PAGE_SIZE;
176 *epc = &sc->epc_pages[i];
177 return (0);
178 }
179
180 return (ENOMEM);
181 }
182
183 static void
sgx_put_epc_page(struct sgx_softc * sc,struct epc_page * epc)184 sgx_put_epc_page(struct sgx_softc *sc, struct epc_page *epc)
185 {
186 vmem_addr_t addr;
187
188 if (epc == NULL)
189 return;
190
191 addr = (epc->index * PAGE_SIZE) + sc->epc_base;
192 vmem_free(sc->vmem_epc, addr, PAGE_SIZE);
193 }
194
195 static int
sgx_va_slot_init_by_index(struct sgx_softc * sc,vm_object_t object,uint64_t idx)196 sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object,
197 uint64_t idx)
198 {
199 struct epc_page *epc;
200 vm_page_t page;
201 vm_page_t p;
202 int ret;
203
204 VM_OBJECT_ASSERT_WLOCKED(object);
205
206 p = vm_page_lookup(object, idx);
207 if (p == NULL) {
208 ret = sgx_get_epc_page(sc, &epc);
209 if (ret) {
210 dprintf("%s: No free EPC pages available.\n",
211 __func__);
212 return (ret);
213 }
214
215 mtx_lock(&sc->mtx_encls);
216 sgx_epa((void *)epc->base);
217 mtx_unlock(&sc->mtx_encls);
218
219 page = PHYS_TO_VM_PAGE(epc->phys);
220
221 page->valid = VM_PAGE_BITS_ALL;
222 vm_page_insert(page, object, idx);
223 }
224
225 return (0);
226 }
227
228 static int
sgx_va_slot_init(struct sgx_softc * sc,struct sgx_enclave * enclave,uint64_t addr)229 sgx_va_slot_init(struct sgx_softc *sc,
230 struct sgx_enclave *enclave,
231 uint64_t addr)
232 {
233 vm_pindex_t pidx;
234 uint64_t va_page_idx;
235 uint64_t idx;
236 vm_object_t object;
237 int ret;
238
239 object = enclave->object;
240
241 VM_OBJECT_ASSERT_WLOCKED(object);
242
243 pidx = OFF_TO_IDX(addr);
244
245 va_page_idx = pidx / SGX_VA_PAGE_SLOTS;
246 idx = - SGX_VA_PAGES_OFFS - va_page_idx;
247
248 ret = sgx_va_slot_init_by_index(sc, object, idx);
249
250 return (ret);
251 }
252
253 static int
sgx_mem_find(struct sgx_softc * sc,uint64_t addr,vm_map_entry_t * entry0,vm_object_t * object0)254 sgx_mem_find(struct sgx_softc *sc, uint64_t addr,
255 vm_map_entry_t *entry0, vm_object_t *object0)
256 {
257 vm_map_t map;
258 vm_map_entry_t entry;
259 vm_object_t object;
260
261 map = &curproc->p_vmspace->vm_map;
262
263 vm_map_lock_read(map);
264 if (!vm_map_lookup_entry(map, addr, &entry)) {
265 vm_map_unlock_read(map);
266 dprintf("%s: Can't find enclave.\n", __func__);
267 return (EINVAL);
268 }
269
270 object = entry->object.vm_object;
271 if (object == NULL || object->handle == NULL) {
272 vm_map_unlock_read(map);
273 return (EINVAL);
274 }
275
276 if (object->type != OBJT_MGTDEVICE ||
277 object->un_pager.devp.ops != &sgx_pg_ops) {
278 vm_map_unlock_read(map);
279 return (EINVAL);
280 }
281
282 vm_object_reference(object);
283
284 *object0 = object;
285 *entry0 = entry;
286 vm_map_unlock_read(map);
287
288 return (0);
289 }
290
291 static int
sgx_enclave_find(struct sgx_softc * sc,uint64_t addr,struct sgx_enclave ** encl)292 sgx_enclave_find(struct sgx_softc *sc, uint64_t addr,
293 struct sgx_enclave **encl)
294 {
295 struct sgx_vm_handle *vmh;
296 struct sgx_enclave *enclave;
297 vm_map_entry_t entry;
298 vm_object_t object;
299 int ret;
300
301 ret = sgx_mem_find(sc, addr, &entry, &object);
302 if (ret)
303 return (ret);
304
305 vmh = object->handle;
306 if (vmh == NULL) {
307 vm_object_deallocate(object);
308 return (EINVAL);
309 }
310
311 enclave = vmh->enclave;
312 if (enclave == NULL || enclave->object == NULL) {
313 vm_object_deallocate(object);
314 return (EINVAL);
315 }
316
317 *encl = enclave;
318
319 return (0);
320 }
321
322 static int
sgx_enclave_alloc(struct sgx_softc * sc,struct secs * secs,struct sgx_enclave ** enclave0)323 sgx_enclave_alloc(struct sgx_softc *sc, struct secs *secs,
324 struct sgx_enclave **enclave0)
325 {
326 struct sgx_enclave *enclave;
327
328 enclave = malloc(sizeof(struct sgx_enclave),
329 M_SGX, M_WAITOK | M_ZERO);
330
331 enclave->base = secs->base;
332 enclave->size = secs->size;
333
334 *enclave0 = enclave;
335
336 return (0);
337 }
338
339 static void
sgx_epc_page_remove(struct sgx_softc * sc,struct epc_page * epc)340 sgx_epc_page_remove(struct sgx_softc *sc,
341 struct epc_page *epc)
342 {
343
344 mtx_lock(&sc->mtx_encls);
345 sgx_eremove((void *)epc->base);
346 mtx_unlock(&sc->mtx_encls);
347 }
348
349 static void
sgx_page_remove(struct sgx_softc * sc,vm_page_t p)350 sgx_page_remove(struct sgx_softc *sc, vm_page_t p)
351 {
352 struct epc_page *epc;
353 vm_paddr_t pa;
354 uint64_t offs;
355
356 (void)vm_page_remove(p);
357
358 dprintf("%s: p->pidx %ld\n", __func__, p->pindex);
359
360 pa = VM_PAGE_TO_PHYS(p);
361 epc = &sc->epc_pages[0];
362 offs = (pa - epc->phys) / PAGE_SIZE;
363 epc = &sc->epc_pages[offs];
364
365 sgx_epc_page_remove(sc, epc);
366 sgx_put_epc_page(sc, epc);
367 }
368
369 static void
sgx_enclave_remove(struct sgx_softc * sc,struct sgx_enclave * enclave)370 sgx_enclave_remove(struct sgx_softc *sc,
371 struct sgx_enclave *enclave)
372 {
373 vm_object_t object;
374 vm_page_t p, p_secs, p_next;
375
376 mtx_lock(&sc->mtx);
377 TAILQ_REMOVE(&sc->enclaves, enclave, next);
378 mtx_unlock(&sc->mtx);
379
380 object = enclave->object;
381
382 VM_OBJECT_WLOCK(object);
383
384 /*
385 * First remove all the pages except SECS,
386 * then remove SECS page.
387 */
388 restart:
389 TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {
390 if (p->pindex == SGX_SECS_VM_OBJECT_INDEX)
391 continue;
392 if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0)
393 goto restart;
394 sgx_page_remove(sc, p);
395 }
396 p_secs = vm_page_grab(object, SGX_SECS_VM_OBJECT_INDEX,
397 VM_ALLOC_NOCREAT);
398 /* Now remove SECS page */
399 if (p_secs != NULL)
400 sgx_page_remove(sc, p_secs);
401
402 KASSERT(TAILQ_EMPTY(&object->memq) == 1, ("not empty"));
403 KASSERT(object->resident_page_count == 0, ("count"));
404
405 VM_OBJECT_WUNLOCK(object);
406 }
407
408 static int
sgx_measure_page(struct sgx_softc * sc,struct epc_page * secs,struct epc_page * epc,uint16_t mrmask)409 sgx_measure_page(struct sgx_softc *sc, struct epc_page *secs,
410 struct epc_page *epc, uint16_t mrmask)
411 {
412 int i, j;
413 int ret;
414
415 mtx_lock(&sc->mtx_encls);
416
417 for (i = 0, j = 1; i < PAGE_SIZE; i += 0x100, j <<= 1) {
418 if (!(j & mrmask))
419 continue;
420
421 ret = sgx_eextend((void *)secs->base,
422 (void *)(epc->base + i));
423 if (ret == SGX_EFAULT) {
424 mtx_unlock(&sc->mtx_encls);
425 return (ret);
426 }
427 }
428
429 mtx_unlock(&sc->mtx_encls);
430
431 return (0);
432 }
433
434 static int
sgx_secs_validate(struct sgx_softc * sc,struct secs * secs)435 sgx_secs_validate(struct sgx_softc *sc, struct secs *secs)
436 {
437 struct secs_attr *attr;
438 int i;
439
440 if (secs->size == 0)
441 return (EINVAL);
442
443 /* BASEADDR must be naturally aligned on an SECS.SIZE boundary. */
444 if (secs->base & (secs->size - 1))
445 return (EINVAL);
446
447 /* SECS.SIZE must be at least 2 pages. */
448 if (secs->size < 2 * PAGE_SIZE)
449 return (EINVAL);
450
451 if ((secs->size & (secs->size - 1)) != 0)
452 return (EINVAL);
453
454 attr = &secs->attributes;
455
456 if (attr->reserved1 != 0 ||
457 attr->reserved2 != 0 ||
458 attr->reserved3 != 0)
459 return (EINVAL);
460
461 for (i = 0; i < SECS_ATTR_RSV4_SIZE; i++)
462 if (attr->reserved4[i])
463 return (EINVAL);
464
465 /*
466 * Intel® Software Guard Extensions Programming Reference
467 * 6.7.2 Relevant Fields in Various Data Structures
468 * 6.7.2.1 SECS.ATTRIBUTES.XFRM
469 * XFRM[1:0] must be set to 0x3.
470 */
471 if ((attr->xfrm & 0x3) != 0x3)
472 return (EINVAL);
473
474 if (!attr->mode64bit)
475 return (EINVAL);
476
477 if (secs->size > sc->enclave_size_max)
478 return (EINVAL);
479
480 for (i = 0; i < SECS_RSV1_SIZE; i++)
481 if (secs->reserved1[i])
482 return (EINVAL);
483
484 for (i = 0; i < SECS_RSV2_SIZE; i++)
485 if (secs->reserved2[i])
486 return (EINVAL);
487
488 for (i = 0; i < SECS_RSV3_SIZE; i++)
489 if (secs->reserved3[i])
490 return (EINVAL);
491
492 for (i = 0; i < SECS_RSV4_SIZE; i++)
493 if (secs->reserved4[i])
494 return (EINVAL);
495
496 return (0);
497 }
498
499 static int
sgx_tcs_validate(struct tcs * tcs)500 sgx_tcs_validate(struct tcs *tcs)
501 {
502 int i;
503
504 if ((tcs->flags) ||
505 (tcs->ossa & (PAGE_SIZE - 1)) ||
506 (tcs->ofsbasgx & (PAGE_SIZE - 1)) ||
507 (tcs->ogsbasgx & (PAGE_SIZE - 1)) ||
508 ((tcs->fslimit & 0xfff) != 0xfff) ||
509 ((tcs->gslimit & 0xfff) != 0xfff))
510 return (EINVAL);
511
512 for (i = 0; i < nitems(tcs->reserved3); i++)
513 if (tcs->reserved3[i])
514 return (EINVAL);
515
516 return (0);
517 }
518
519 static void
sgx_tcs_dump(struct sgx_softc * sc,struct tcs * t)520 sgx_tcs_dump(struct sgx_softc *sc, struct tcs *t)
521 {
522
523 dprintf("t->flags %lx\n", t->flags);
524 dprintf("t->ossa %lx\n", t->ossa);
525 dprintf("t->cssa %x\n", t->cssa);
526 dprintf("t->nssa %x\n", t->nssa);
527 dprintf("t->oentry %lx\n", t->oentry);
528 dprintf("t->ofsbasgx %lx\n", t->ofsbasgx);
529 dprintf("t->ogsbasgx %lx\n", t->ogsbasgx);
530 dprintf("t->fslimit %x\n", t->fslimit);
531 dprintf("t->gslimit %x\n", t->gslimit);
532 }
533
534 static int
sgx_pg_ctor(void * handle,vm_ooffset_t size,vm_prot_t prot,vm_ooffset_t foff,struct ucred * cred,u_short * color)535 sgx_pg_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
536 vm_ooffset_t foff, struct ucred *cred, u_short *color)
537 {
538 struct sgx_vm_handle *vmh;
539
540 vmh = handle;
541 if (vmh == NULL) {
542 dprintf("%s: vmh not found.\n", __func__);
543 return (0);
544 }
545
546 dprintf("%s: vmh->base %lx foff 0x%lx size 0x%lx\n",
547 __func__, vmh->base, foff, size);
548
549 return (0);
550 }
551
552 static void
sgx_pg_dtor(void * handle)553 sgx_pg_dtor(void *handle)
554 {
555 struct sgx_vm_handle *vmh;
556 struct sgx_softc *sc;
557
558 vmh = handle;
559 if (vmh == NULL) {
560 dprintf("%s: vmh not found.\n", __func__);
561 return;
562 }
563
564 sc = vmh->sc;
565 if (sc == NULL) {
566 dprintf("%s: sc is NULL\n", __func__);
567 return;
568 }
569
570 if (vmh->enclave == NULL) {
571 dprintf("%s: Enclave not found.\n", __func__);
572 return;
573 }
574
575 sgx_enclave_remove(sc, vmh->enclave);
576
577 free(vmh->enclave, M_SGX);
578 free(vmh, M_SGX);
579 }
580
581 static int
sgx_pg_fault(vm_object_t object,vm_ooffset_t offset,int prot,vm_page_t * mres)582 sgx_pg_fault(vm_object_t object, vm_ooffset_t offset,
583 int prot, vm_page_t *mres)
584 {
585
586 /*
587 * The purpose of this trivial handler is to handle the race
588 * when user tries to access mmaped region before or during
589 * enclave creation ioctl calls.
590 */
591
592 dprintf("%s: offset 0x%lx\n", __func__, offset);
593
594 return (VM_PAGER_FAIL);
595 }
596
597 static struct cdev_pager_ops sgx_pg_ops = {
598 .cdev_pg_ctor = sgx_pg_ctor,
599 .cdev_pg_dtor = sgx_pg_dtor,
600 .cdev_pg_fault = sgx_pg_fault,
601 };
602
603 static void
sgx_insert_epc_page_by_index(vm_page_t page,vm_object_t object,vm_pindex_t pidx)604 sgx_insert_epc_page_by_index(vm_page_t page, vm_object_t object,
605 vm_pindex_t pidx)
606 {
607
608 VM_OBJECT_ASSERT_WLOCKED(object);
609
610 page->valid = VM_PAGE_BITS_ALL;
611 vm_page_insert(page, object, pidx);
612 }
613
614 static void
sgx_insert_epc_page(struct sgx_enclave * enclave,struct epc_page * epc,uint64_t addr)615 sgx_insert_epc_page(struct sgx_enclave *enclave,
616 struct epc_page *epc, uint64_t addr)
617 {
618 vm_pindex_t pidx;
619 vm_page_t page;
620
621 VM_OBJECT_ASSERT_WLOCKED(enclave->object);
622
623 pidx = OFF_TO_IDX(addr);
624 page = PHYS_TO_VM_PAGE(epc->phys);
625
626 sgx_insert_epc_page_by_index(page, enclave->object, pidx);
627 }
628
629 static int
sgx_ioctl_create(struct sgx_softc * sc,struct sgx_enclave_create * param)630 sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
631 {
632 struct sgx_vm_handle *vmh;
633 vm_map_entry_t entry;
634 vm_page_t p;
635 struct page_info pginfo;
636 struct secinfo secinfo;
637 struct sgx_enclave *enclave;
638 struct epc_page *epc;
639 struct secs *secs;
640 vm_object_t object;
641 vm_page_t page;
642 int ret;
643
644 epc = NULL;
645 secs = NULL;
646 enclave = NULL;
647 object = NULL;
648
649 /* SGX Enclave Control Structure (SECS) */
650 secs = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO);
651 ret = copyin((void *)param->src, secs, sizeof(struct secs));
652 if (ret) {
653 dprintf("%s: Can't copy SECS.\n", __func__);
654 goto error;
655 }
656
657 ret = sgx_secs_validate(sc, secs);
658 if (ret) {
659 dprintf("%s: SECS validation failed.\n", __func__);
660 goto error;
661 }
662
663 ret = sgx_mem_find(sc, secs->base, &entry, &object);
664 if (ret) {
665 dprintf("%s: Can't find vm_map.\n", __func__);
666 goto error;
667 }
668
669 vmh = object->handle;
670 if (!vmh) {
671 dprintf("%s: Can't find vmh.\n", __func__);
672 ret = ENXIO;
673 goto error;
674 }
675
676 dprintf("%s: entry start %lx offset %lx\n",
677 __func__, entry->start, entry->offset);
678 vmh->base = (entry->start - entry->offset);
679
680 ret = sgx_enclave_alloc(sc, secs, &enclave);
681 if (ret) {
682 dprintf("%s: Can't alloc enclave.\n", __func__);
683 goto error;
684 }
685 enclave->object = object;
686 enclave->vmh = vmh;
687
688 memset(&secinfo, 0, sizeof(struct secinfo));
689 memset(&pginfo, 0, sizeof(struct page_info));
690 pginfo.linaddr = 0;
691 pginfo.srcpge = (uint64_t)secs;
692 pginfo.secinfo = &secinfo;
693 pginfo.secs = 0;
694
695 ret = sgx_get_epc_page(sc, &epc);
696 if (ret) {
697 dprintf("%s: Failed to get free epc page.\n", __func__);
698 goto error;
699 }
700 enclave->secs_epc_page = epc;
701
702 VM_OBJECT_WLOCK(object);
703 p = vm_page_lookup(object, SGX_SECS_VM_OBJECT_INDEX);
704 if (p) {
705 VM_OBJECT_WUNLOCK(object);
706 /* SECS page already added. */
707 ret = ENXIO;
708 goto error;
709 }
710
711 ret = sgx_va_slot_init_by_index(sc, object,
712 - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX);
713 if (ret) {
714 VM_OBJECT_WUNLOCK(object);
715 dprintf("%s: Can't init va slot.\n", __func__);
716 goto error;
717 }
718
719 mtx_lock(&sc->mtx);
720 if ((sc->state & SGX_STATE_RUNNING) == 0) {
721 mtx_unlock(&sc->mtx);
722 /* Remove VA page that was just created for SECS page. */
723 p = vm_page_grab(enclave->object,
724 - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX,
725 VM_ALLOC_NOCREAT);
726 sgx_page_remove(sc, p);
727 VM_OBJECT_WUNLOCK(object);
728 goto error;
729 }
730 mtx_lock(&sc->mtx_encls);
731 ret = sgx_ecreate(&pginfo, (void *)epc->base);
732 mtx_unlock(&sc->mtx_encls);
733 if (ret == SGX_EFAULT) {
734 dprintf("%s: gp fault\n", __func__);
735 mtx_unlock(&sc->mtx);
736 /* Remove VA page that was just created for SECS page. */
737 p = vm_page_grab(enclave->object,
738 - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX,
739 VM_ALLOC_NOCREAT);
740 sgx_page_remove(sc, p);
741 VM_OBJECT_WUNLOCK(object);
742 goto error;
743 }
744
745 TAILQ_INSERT_TAIL(&sc->enclaves, enclave, next);
746 mtx_unlock(&sc->mtx);
747
748 vmh->enclave = enclave;
749
750 page = PHYS_TO_VM_PAGE(epc->phys);
751 sgx_insert_epc_page_by_index(page, enclave->object,
752 SGX_SECS_VM_OBJECT_INDEX);
753
754 VM_OBJECT_WUNLOCK(object);
755
756 /* Release the reference. */
757 vm_object_deallocate(object);
758
759 free(secs, M_SGX);
760
761 return (0);
762
763 error:
764 free(secs, M_SGX);
765 sgx_put_epc_page(sc, epc);
766 free(enclave, M_SGX);
767 vm_object_deallocate(object);
768
769 return (ret);
770 }
771
772 static int
sgx_ioctl_add_page(struct sgx_softc * sc,struct sgx_enclave_add_page * addp)773 sgx_ioctl_add_page(struct sgx_softc *sc,
774 struct sgx_enclave_add_page *addp)
775 {
776 struct epc_page *secs_epc_page;
777 struct sgx_enclave *enclave;
778 struct sgx_vm_handle *vmh;
779 struct epc_page *epc;
780 struct page_info pginfo;
781 struct secinfo secinfo;
782 vm_object_t object;
783 void *tmp_vaddr;
784 uint64_t page_type;
785 struct tcs *t;
786 uint64_t addr;
787 uint64_t pidx;
788 vm_page_t p;
789 int ret;
790
791 tmp_vaddr = NULL;
792 epc = NULL;
793 object = NULL;
794
795 /* Find and get reference to VM object. */
796 ret = sgx_enclave_find(sc, addp->addr, &enclave);
797 if (ret) {
798 dprintf("%s: Failed to find enclave.\n", __func__);
799 goto error;
800 }
801
802 object = enclave->object;
803 KASSERT(object != NULL, ("vm object is NULL\n"));
804 vmh = object->handle;
805
806 ret = sgx_get_epc_page(sc, &epc);
807 if (ret) {
808 dprintf("%s: Failed to get free epc page.\n", __func__);
809 goto error;
810 }
811
812 memset(&secinfo, 0, sizeof(struct secinfo));
813 ret = copyin((void *)addp->secinfo, &secinfo,
814 sizeof(struct secinfo));
815 if (ret) {
816 dprintf("%s: Failed to copy secinfo.\n", __func__);
817 goto error;
818 }
819
820 tmp_vaddr = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO);
821 ret = copyin((void *)addp->src, tmp_vaddr, PAGE_SIZE);
822 if (ret) {
823 dprintf("%s: Failed to copy page.\n", __func__);
824 goto error;
825 }
826
827 page_type = (secinfo.flags & SECINFO_FLAGS_PT_M) >>
828 SECINFO_FLAGS_PT_S;
829 if (page_type != SGX_PT_TCS && page_type != SGX_PT_REG) {
830 dprintf("%s: page can't be added.\n", __func__);
831 goto error;
832 }
833 if (page_type == SGX_PT_TCS) {
834 t = (struct tcs *)tmp_vaddr;
835 ret = sgx_tcs_validate(t);
836 if (ret) {
837 dprintf("%s: TCS page validation failed.\n",
838 __func__);
839 goto error;
840 }
841 sgx_tcs_dump(sc, t);
842 }
843
844 addr = (addp->addr - vmh->base);
845 pidx = OFF_TO_IDX(addr);
846
847 VM_OBJECT_WLOCK(object);
848 p = vm_page_lookup(object, pidx);
849 if (p) {
850 VM_OBJECT_WUNLOCK(object);
851 /* Page already added. */
852 ret = ENXIO;
853 goto error;
854 }
855
856 ret = sgx_va_slot_init(sc, enclave, addr);
857 if (ret) {
858 VM_OBJECT_WUNLOCK(object);
859 dprintf("%s: Can't init va slot.\n", __func__);
860 goto error;
861 }
862
863 secs_epc_page = enclave->secs_epc_page;
864 memset(&pginfo, 0, sizeof(struct page_info));
865 pginfo.linaddr = (uint64_t)addp->addr;
866 pginfo.srcpge = (uint64_t)tmp_vaddr;
867 pginfo.secinfo = &secinfo;
868 pginfo.secs = (uint64_t)secs_epc_page->base;
869
870 mtx_lock(&sc->mtx_encls);
871 ret = sgx_eadd(&pginfo, (void *)epc->base);
872 if (ret == SGX_EFAULT) {
873 dprintf("%s: gp fault on eadd\n", __func__);
874 mtx_unlock(&sc->mtx_encls);
875 VM_OBJECT_WUNLOCK(object);
876 goto error;
877 }
878 mtx_unlock(&sc->mtx_encls);
879
880 ret = sgx_measure_page(sc, enclave->secs_epc_page, epc, addp->mrmask);
881 if (ret == SGX_EFAULT) {
882 dprintf("%s: gp fault on eextend\n", __func__);
883 sgx_epc_page_remove(sc, epc);
884 VM_OBJECT_WUNLOCK(object);
885 goto error;
886 }
887
888 sgx_insert_epc_page(enclave, epc, addr);
889
890 VM_OBJECT_WUNLOCK(object);
891
892 /* Release the reference. */
893 vm_object_deallocate(object);
894
895 free(tmp_vaddr, M_SGX);
896
897 return (0);
898
899 error:
900 free(tmp_vaddr, M_SGX);
901 sgx_put_epc_page(sc, epc);
902 vm_object_deallocate(object);
903
904 return (ret);
905 }
906
907 static int
sgx_ioctl_init(struct sgx_softc * sc,struct sgx_enclave_init * initp)908 sgx_ioctl_init(struct sgx_softc *sc, struct sgx_enclave_init *initp)
909 {
910 struct epc_page *secs_epc_page;
911 struct sgx_enclave *enclave;
912 struct thread *td;
913 void *tmp_vaddr;
914 void *einittoken;
915 void *sigstruct;
916 vm_object_t object;
917 int retry;
918 int ret;
919
920 td = curthread;
921 tmp_vaddr = NULL;
922 object = NULL;
923
924 dprintf("%s: addr %lx, sigstruct %lx, einittoken %lx\n",
925 __func__, initp->addr, initp->sigstruct, initp->einittoken);
926
927 /* Find and get reference to VM object. */
928 ret = sgx_enclave_find(sc, initp->addr, &enclave);
929 if (ret) {
930 dprintf("%s: Failed to find enclave.\n", __func__);
931 goto error;
932 }
933
934 object = enclave->object;
935
936 tmp_vaddr = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO);
937 sigstruct = tmp_vaddr;
938 einittoken = (void *)((uint64_t)sigstruct + PAGE_SIZE / 2);
939
940 ret = copyin((void *)initp->sigstruct, sigstruct,
941 SGX_SIGSTRUCT_SIZE);
942 if (ret) {
943 dprintf("%s: Failed to copy SIGSTRUCT page.\n", __func__);
944 goto error;
945 }
946
947 ret = copyin((void *)initp->einittoken, einittoken,
948 SGX_EINITTOKEN_SIZE);
949 if (ret) {
950 dprintf("%s: Failed to copy EINITTOKEN page.\n", __func__);
951 goto error;
952 }
953
954 secs_epc_page = enclave->secs_epc_page;
955 retry = 16;
956 do {
957 mtx_lock(&sc->mtx_encls);
958 ret = sgx_einit(sigstruct, (void *)secs_epc_page->base,
959 einittoken);
960 mtx_unlock(&sc->mtx_encls);
961 dprintf("%s: sgx_einit returned %d\n", __func__, ret);
962 } while (ret == SGX_UNMASKED_EVENT && retry--);
963
964 if (ret) {
965 dprintf("%s: Failed init enclave: %d\n", __func__, ret);
966 td->td_retval[0] = ret;
967 ret = 0;
968 }
969
970 error:
971 free(tmp_vaddr, M_SGX);
972
973 /* Release the reference. */
974 vm_object_deallocate(object);
975
976 return (ret);
977 }
978
979 static int
sgx_ioctl(struct cdev * dev,u_long cmd,caddr_t addr,int flags,struct thread * td)980 sgx_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
981 struct thread *td)
982 {
983 struct sgx_enclave_add_page *addp;
984 struct sgx_enclave_create *param;
985 struct sgx_enclave_init *initp;
986 struct sgx_softc *sc;
987 int ret;
988 int len;
989
990 sc = &sgx_sc;
991
992 len = IOCPARM_LEN(cmd);
993
994 dprintf("%s: cmd %lx, addr %lx, len %d\n",
995 __func__, cmd, (uint64_t)addr, len);
996
997 if (len > SGX_IOCTL_MAX_DATA_LEN)
998 return (EINVAL);
999
1000 switch (cmd) {
1001 case SGX_IOC_ENCLAVE_CREATE:
1002 param = (struct sgx_enclave_create *)addr;
1003 ret = sgx_ioctl_create(sc, param);
1004 break;
1005 case SGX_IOC_ENCLAVE_ADD_PAGE:
1006 addp = (struct sgx_enclave_add_page *)addr;
1007 ret = sgx_ioctl_add_page(sc, addp);
1008 break;
1009 case SGX_IOC_ENCLAVE_INIT:
1010 initp = (struct sgx_enclave_init *)addr;
1011 ret = sgx_ioctl_init(sc, initp);
1012 break;
1013 default:
1014 return (EINVAL);
1015 }
1016
1017 return (ret);
1018 }
1019
1020 static int
sgx_mmap_single(struct cdev * cdev,vm_ooffset_t * offset,vm_size_t mapsize,struct vm_object ** objp,int nprot)1021 sgx_mmap_single(struct cdev *cdev, vm_ooffset_t *offset,
1022 vm_size_t mapsize, struct vm_object **objp, int nprot)
1023 {
1024 struct sgx_vm_handle *vmh;
1025 struct sgx_softc *sc;
1026
1027 sc = &sgx_sc;
1028
1029 dprintf("%s: mapsize 0x%lx, offset %lx\n",
1030 __func__, mapsize, *offset);
1031
1032 vmh = malloc(sizeof(struct sgx_vm_handle),
1033 M_SGX, M_WAITOK | M_ZERO);
1034 vmh->sc = sc;
1035 vmh->size = mapsize;
1036 vmh->mem = cdev_pager_allocate(vmh, OBJT_MGTDEVICE, &sgx_pg_ops,
1037 mapsize, nprot, *offset, NULL);
1038 if (vmh->mem == NULL) {
1039 free(vmh, M_SGX);
1040 return (ENOMEM);
1041 }
1042
1043 VM_OBJECT_WLOCK(vmh->mem);
1044 vm_object_set_flag(vmh->mem, OBJ_PG_DTOR);
1045 VM_OBJECT_WUNLOCK(vmh->mem);
1046
1047 *objp = vmh->mem;
1048
1049 return (0);
1050 }
1051
1052 static struct cdevsw sgx_cdevsw = {
1053 .d_version = D_VERSION,
1054 .d_ioctl = sgx_ioctl,
1055 .d_mmap_single = sgx_mmap_single,
1056 .d_name = "Intel SGX",
1057 };
1058
1059 static int
sgx_get_epc_area(struct sgx_softc * sc)1060 sgx_get_epc_area(struct sgx_softc *sc)
1061 {
1062 vm_offset_t epc_base_vaddr;
1063 u_int cp[4];
1064 int error;
1065 int i;
1066
1067 cpuid_count(SGX_CPUID, 0x2, cp);
1068
1069 sc->epc_base = ((uint64_t)(cp[1] & 0xfffff) << 32) +
1070 (cp[0] & 0xfffff000);
1071 sc->epc_size = ((uint64_t)(cp[3] & 0xfffff) << 32) +
1072 (cp[2] & 0xfffff000);
1073 sc->npages = sc->epc_size / SGX_PAGE_SIZE;
1074
1075 if (sc->epc_size == 0 || sc->epc_base == 0) {
1076 printf("%s: Incorrect EPC data: EPC base %lx, size %lu\n",
1077 __func__, sc->epc_base, sc->epc_size);
1078 return (EINVAL);
1079 }
1080
1081 if (cp[3] & 0xffff)
1082 sc->enclave_size_max = (1 << ((cp[3] >> 8) & 0xff));
1083 else
1084 sc->enclave_size_max = SGX_ENCL_SIZE_MAX_DEF;
1085
1086 epc_base_vaddr = (vm_offset_t)pmap_mapdev_attr(sc->epc_base,
1087 sc->epc_size, VM_MEMATTR_DEFAULT);
1088
1089 sc->epc_pages = malloc(sizeof(struct epc_page) * sc->npages,
1090 M_DEVBUF, M_WAITOK | M_ZERO);
1091
1092 for (i = 0; i < sc->npages; i++) {
1093 sc->epc_pages[i].base = epc_base_vaddr + SGX_PAGE_SIZE * i;
1094 sc->epc_pages[i].phys = sc->epc_base + SGX_PAGE_SIZE * i;
1095 sc->epc_pages[i].index = i;
1096 }
1097
1098 sc->vmem_epc = vmem_create("SGX EPC", sc->epc_base, sc->epc_size,
1099 PAGE_SIZE, PAGE_SIZE, M_FIRSTFIT | M_WAITOK);
1100 if (sc->vmem_epc == NULL) {
1101 printf("%s: Can't create vmem arena.\n", __func__);
1102 free(sc->epc_pages, M_SGX);
1103 return (EINVAL);
1104 }
1105
1106 error = vm_phys_fictitious_reg_range(sc->epc_base,
1107 sc->epc_base + sc->epc_size, VM_MEMATTR_DEFAULT);
1108 if (error) {
1109 printf("%s: Can't register fictitious space.\n", __func__);
1110 free(sc->epc_pages, M_SGX);
1111 return (EINVAL);
1112 }
1113
1114 return (0);
1115 }
1116
1117 static void
sgx_put_epc_area(struct sgx_softc * sc)1118 sgx_put_epc_area(struct sgx_softc *sc)
1119 {
1120
1121 vm_phys_fictitious_unreg_range(sc->epc_base,
1122 sc->epc_base + sc->epc_size);
1123
1124 free(sc->epc_pages, M_SGX);
1125 }
1126
1127 static int
sgx_load(void)1128 sgx_load(void)
1129 {
1130 struct sgx_softc *sc;
1131 int error;
1132
1133 sc = &sgx_sc;
1134
1135 if ((cpu_stdext_feature & CPUID_STDEXT_SGX) == 0)
1136 return (ENXIO);
1137
1138 error = sgx_get_epc_area(sc);
1139 if (error) {
1140 printf("%s: Failed to get Processor Reserved Memory area.\n",
1141 __func__);
1142 return (ENXIO);
1143 }
1144
1145 mtx_init(&sc->mtx_encls, "SGX ENCLS", NULL, MTX_DEF);
1146 mtx_init(&sc->mtx, "SGX driver", NULL, MTX_DEF);
1147
1148 TAILQ_INIT(&sc->enclaves);
1149
1150 sc->sgx_cdev = make_dev(&sgx_cdevsw, 0, UID_ROOT, GID_WHEEL,
1151 0600, "isgx");
1152
1153 sc->state |= SGX_STATE_RUNNING;
1154
1155 printf("SGX initialized: EPC base 0x%lx size %ld (%d pages)\n",
1156 sc->epc_base, sc->epc_size, sc->npages);
1157
1158 return (0);
1159 }
1160
1161 static int
sgx_unload(void)1162 sgx_unload(void)
1163 {
1164 struct sgx_softc *sc;
1165
1166 sc = &sgx_sc;
1167
1168 if ((sc->state & SGX_STATE_RUNNING) == 0)
1169 return (0);
1170
1171 mtx_lock(&sc->mtx);
1172 if (!TAILQ_EMPTY(&sc->enclaves)) {
1173 mtx_unlock(&sc->mtx);
1174 return (EBUSY);
1175 }
1176 sc->state &= ~SGX_STATE_RUNNING;
1177 mtx_unlock(&sc->mtx);
1178
1179 destroy_dev(sc->sgx_cdev);
1180
1181 vmem_destroy(sc->vmem_epc);
1182 sgx_put_epc_area(sc);
1183
1184 mtx_destroy(&sc->mtx_encls);
1185 mtx_destroy(&sc->mtx);
1186
1187 return (0);
1188 }
1189
1190 static int
sgx_handler(module_t mod,int what,void * arg)1191 sgx_handler(module_t mod, int what, void *arg)
1192 {
1193 int error;
1194
1195 switch (what) {
1196 case MOD_LOAD:
1197 error = sgx_load();
1198 break;
1199 case MOD_UNLOAD:
1200 error = sgx_unload();
1201 break;
1202 default:
1203 error = 0;
1204 break;
1205 }
1206
1207 return (error);
1208 }
1209
1210 static moduledata_t sgx_kmod = {
1211 "sgx",
1212 sgx_handler,
1213 NULL
1214 };
1215
1216 DECLARE_MODULE(sgx, sgx_kmod, SI_SUB_LAST, SI_ORDER_ANY);
1217 MODULE_VERSION(sgx, 1);
1218