1 /*-
2 * Copyright (c) 2016 Akshay Jaggi <[email protected]>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * gntdev.c
27 *
28 * Interface to /dev/xen/gntdev.
29 *
30 */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/uio.h>
38 #include <sys/bus.h>
39 #include <sys/malloc.h>
40 #include <sys/kernel.h>
41 #include <sys/lock.h>
42 #include <sys/mutex.h>
43 #include <sys/rwlock.h>
44 #include <sys/selinfo.h>
45 #include <sys/poll.h>
46 #include <sys/conf.h>
47 #include <sys/fcntl.h>
48 #include <sys/ioccom.h>
49 #include <sys/rman.h>
50 #include <sys/tree.h>
51 #include <sys/module.h>
52 #include <sys/proc.h>
53 #include <sys/bitset.h>
54 #include <sys/queue.h>
55 #include <sys/mman.h>
56 #include <sys/syslog.h>
57 #include <sys/taskqueue.h>
58
59 #include <vm/vm.h>
60 #include <vm/vm_param.h>
61 #include <vm/vm_extern.h>
62 #include <vm/vm_kern.h>
63 #include <vm/vm_page.h>
64 #include <vm/vm_map.h>
65 #include <vm/vm_object.h>
66 #include <vm/vm_pager.h>
67
68 #include <machine/md_var.h>
69
70 #include <xen/xen-os.h>
71 #include <xen/hypervisor.h>
72 #include <xen/error.h>
73 #include <xen/xen_intr.h>
74 #include <xen/gnttab.h>
75 #include <xen/gntdev.h>
76
77 MALLOC_DEFINE(M_GNTDEV, "gntdev", "Xen grant-table user-space device");
78
79 #define MAX_OFFSET_COUNT ((0xffffffffffffffffull >> PAGE_SHIFT) + 1)
80
81 static d_open_t gntdev_open;
82 static d_ioctl_t gntdev_ioctl;
83 static d_mmap_single_t gntdev_mmap_single;
84
85 static struct cdevsw gntdev_devsw = {
86 .d_version = D_VERSION,
87 .d_open = gntdev_open,
88 .d_ioctl = gntdev_ioctl,
89 .d_mmap_single = gntdev_mmap_single,
90 .d_name = "gntdev",
91 };
92
93 static device_t gntdev_dev = NULL;
94
95 struct gntdev_gref;
96 struct gntdev_gmap;
97 STAILQ_HEAD(gref_list_head, gntdev_gref);
98 STAILQ_HEAD(gmap_list_head, gntdev_gmap);
99 RB_HEAD(gref_tree_head, gntdev_gref);
100 RB_HEAD(gmap_tree_head, gntdev_gmap);
101
102 struct file_offset_struct {
103 RB_ENTRY(file_offset_struct) next;
104 uint64_t file_offset;
105 uint64_t count;
106 };
107
108 static int
offset_cmp(struct file_offset_struct * f1,struct file_offset_struct * f2)109 offset_cmp(struct file_offset_struct *f1, struct file_offset_struct *f2)
110 {
111 return (f1->file_offset - f2->file_offset);
112 }
113
114 RB_HEAD(file_offset_head, file_offset_struct);
115 RB_GENERATE_STATIC(file_offset_head, file_offset_struct, next, offset_cmp);
116
117 struct per_user_data {
118 struct mtx user_data_lock;
119 struct gref_tree_head gref_tree;
120 struct gmap_tree_head gmap_tree;
121 struct file_offset_head file_offset;
122 };
123
124 /*
125 * Get offset into the file which will be used while mmapping the
126 * appropriate pages by the userspace program.
127 */
128 static int
get_file_offset(struct per_user_data * priv_user,uint32_t count,uint64_t * file_offset)129 get_file_offset(struct per_user_data *priv_user, uint32_t count,
130 uint64_t *file_offset)
131 {
132 struct file_offset_struct *offset, *offset_tmp;
133
134 if (count == 0)
135 return (EINVAL);
136 mtx_lock(&priv_user->user_data_lock);
137 RB_FOREACH_SAFE(offset, file_offset_head, &priv_user->file_offset,
138 offset_tmp) {
139 if (offset->count >= count) {
140 offset->count -= count;
141 *file_offset = offset->file_offset + offset->count *
142 PAGE_SIZE;
143 if (offset->count == 0) {
144 RB_REMOVE(file_offset_head,
145 &priv_user->file_offset, offset);
146 free(offset, M_GNTDEV);
147 }
148 mtx_unlock(&priv_user->user_data_lock);
149 return (0);
150 }
151 }
152 mtx_unlock(&priv_user->user_data_lock);
153
154 return (ENOSPC);
155 }
156
157 static void
put_file_offset(struct per_user_data * priv_user,uint32_t count,uint64_t file_offset)158 put_file_offset(struct per_user_data *priv_user, uint32_t count,
159 uint64_t file_offset)
160 {
161 struct file_offset_struct *offset, *offset_nxt, *offset_prv;
162
163 offset = malloc(sizeof(*offset), M_GNTDEV, M_WAITOK | M_ZERO);
164 offset->file_offset = file_offset;
165 offset->count = count;
166
167 mtx_lock(&priv_user->user_data_lock);
168 RB_INSERT(file_offset_head, &priv_user->file_offset, offset);
169 offset_nxt = RB_NEXT(file_offset_head, &priv_user->file_offset, offset);
170 offset_prv = RB_PREV(file_offset_head, &priv_user->file_offset, offset);
171 if (offset_nxt != NULL &&
172 offset_nxt->file_offset == offset->file_offset + offset->count *
173 PAGE_SIZE) {
174 offset->count += offset_nxt->count;
175 RB_REMOVE(file_offset_head, &priv_user->file_offset,
176 offset_nxt);
177 free(offset_nxt, M_GNTDEV);
178 }
179 if (offset_prv != NULL &&
180 offset->file_offset == offset_prv->file_offset + offset_prv->count *
181 PAGE_SIZE) {
182 offset_prv->count += offset->count;
183 RB_REMOVE(file_offset_head, &priv_user->file_offset, offset);
184 free(offset, M_GNTDEV);
185 }
186 mtx_unlock(&priv_user->user_data_lock);
187 }
188
189 static int gntdev_gmap_pg_ctor(void *handle, vm_ooffset_t size,
190 vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred, u_short *color);
191 static void gntdev_gmap_pg_dtor(void *handle);
192 static int gntdev_gmap_pg_fault(vm_object_t object, vm_ooffset_t offset,
193 int prot, vm_page_t *mres);
194
195 static struct cdev_pager_ops gntdev_gmap_pg_ops = {
196 .cdev_pg_fault = gntdev_gmap_pg_fault,
197 .cdev_pg_ctor = gntdev_gmap_pg_ctor,
198 .cdev_pg_dtor = gntdev_gmap_pg_dtor,
199 };
200
201 struct cleanup_data_struct {
202 struct mtx to_kill_grefs_mtx;
203 struct mtx to_kill_gmaps_mtx;
204 struct gref_list_head to_kill_grefs;
205 struct gmap_list_head to_kill_gmaps;
206 };
207
208 static struct cleanup_data_struct cleanup_data = {
209 .to_kill_grefs = STAILQ_HEAD_INITIALIZER(cleanup_data.to_kill_grefs),
210 .to_kill_gmaps = STAILQ_HEAD_INITIALIZER(cleanup_data.to_kill_gmaps),
211 };
212 MTX_SYSINIT(to_kill_grefs_mtx, &cleanup_data.to_kill_grefs_mtx,
213 "gntdev to_kill_grefs mutex", MTX_DEF);
214 MTX_SYSINIT(to_kill_gmaps_mtx, &cleanup_data.to_kill_gmaps_mtx,
215 "gntdev to_kill_gmaps mutex", MTX_DEF);
216
217 static void cleanup_function(void *arg, __unused int pending);
218 static struct task cleanup_task = TASK_INITIALIZER(0, cleanup_function,
219 &cleanup_data);
220
221 struct notify_data {
222 uint64_t index;
223 uint32_t action;
224 uint32_t event_channel_port;
225 xen_intr_handle_t notify_evtchn_handle;
226 };
227
228 static void notify(struct notify_data *notify, vm_page_t page);
229
230 /*-------------------- Grant Allocation Methods -----------------------------*/
231
232 struct gntdev_gref {
233 union gref_next_union {
234 STAILQ_ENTRY(gntdev_gref) list;
235 RB_ENTRY(gntdev_gref) tree;
236 } gref_next;
237 uint64_t file_index;
238 grant_ref_t gref_id;
239 vm_page_t page;
240 struct notify_data *notify;
241 };
242
243 static int
gref_cmp(struct gntdev_gref * g1,struct gntdev_gref * g2)244 gref_cmp(struct gntdev_gref *g1, struct gntdev_gref *g2)
245 {
246 return (g1->file_index - g2->file_index);
247 }
248
249 RB_GENERATE_STATIC(gref_tree_head, gntdev_gref, gref_next.tree, gref_cmp);
250
251 /*
252 * Traverse over the device-list of to-be-deleted grants allocated, and
253 * if all accesses, both local mmaps and foreign maps, to them have ended,
254 * destroy them.
255 */
256 static void
gref_list_dtor(struct cleanup_data_struct * cleanup_data)257 gref_list_dtor(struct cleanup_data_struct *cleanup_data)
258 {
259 struct gref_list_head tmp_grefs;
260 struct gntdev_gref *gref, *gref_tmp, *gref_previous;
261
262 STAILQ_INIT(&tmp_grefs);
263 mtx_lock(&cleanup_data->to_kill_grefs_mtx);
264 STAILQ_SWAP(&cleanup_data->to_kill_grefs, &tmp_grefs, gntdev_gref);
265 mtx_unlock(&cleanup_data->to_kill_grefs_mtx);
266
267 gref_previous = NULL;
268 STAILQ_FOREACH_SAFE(gref, &tmp_grefs, gref_next.list, gref_tmp) {
269 if (gref->page && gref->page->object == NULL) {
270 if (gref->notify) {
271 notify(gref->notify, gref->page);
272 }
273 if (gref->gref_id != GRANT_REF_INVALID) {
274 if (gnttab_query_foreign_access(gref->gref_id))
275 continue;
276 if (gnttab_end_foreign_access_ref(gref->gref_id)
277 == 0)
278 continue;
279 gnttab_free_grant_reference(gref->gref_id);
280 }
281 vm_page_unwire_noq(gref->page);
282 vm_page_free(gref->page);
283 gref->page = NULL;
284 }
285 if (gref->page == NULL) {
286 if (gref_previous == NULL)
287 STAILQ_REMOVE_HEAD(&tmp_grefs, gref_next.list);
288 else
289 STAILQ_REMOVE_AFTER(&tmp_grefs, gref_previous,
290 gref_next.list);
291 if (gref->notify)
292 free(gref->notify, M_GNTDEV);
293 free(gref, M_GNTDEV);
294 }
295 else
296 gref_previous = gref;
297 }
298
299 if (!STAILQ_EMPTY(&tmp_grefs)) {
300 mtx_lock(&cleanup_data->to_kill_grefs_mtx);
301 STAILQ_CONCAT(&cleanup_data->to_kill_grefs, &tmp_grefs);
302 mtx_unlock(&cleanup_data->to_kill_grefs_mtx);
303 }
304 }
305
306 /*
307 * Find count number of contiguous allocated grants for a given userspace
308 * program by file-offset (index).
309 */
310 static struct gntdev_gref*
gntdev_find_grefs(struct per_user_data * priv_user,uint64_t index,uint32_t count)311 gntdev_find_grefs(struct per_user_data *priv_user,
312 uint64_t index, uint32_t count)
313 {
314 struct gntdev_gref find_gref, *gref, *gref_start = NULL;
315
316 find_gref.file_index = index;
317
318 mtx_lock(&priv_user->user_data_lock);
319 gref_start = RB_FIND(gref_tree_head, &priv_user->gref_tree, &find_gref);
320 for (gref = gref_start; gref != NULL && count > 0; gref =
321 RB_NEXT(gref_tree_head, &priv_user->gref_tree, gref)) {
322 if (index != gref->file_index)
323 break;
324 index += PAGE_SIZE;
325 count--;
326 }
327 mtx_unlock(&priv_user->user_data_lock);
328
329 if (count)
330 return (NULL);
331 return (gref_start);
332 }
333
334 /*
335 * IOCTL_GNTDEV_ALLOC_GREF
336 * Allocate required number of wired pages for the request, grant foreign
337 * access to the physical frames for these pages, and add details about
338 * this allocation to the per user private data, so that these pages can
339 * be mmapped by the userspace program.
340 */
341 static int
gntdev_alloc_gref(struct ioctl_gntdev_alloc_gref * arg)342 gntdev_alloc_gref(struct ioctl_gntdev_alloc_gref *arg)
343 {
344 uint32_t i;
345 int error, readonly;
346 uint64_t file_offset;
347 struct gntdev_gref *grefs;
348 struct per_user_data *priv_user;
349
350 readonly = !(arg->flags & GNTDEV_ALLOC_FLAG_WRITABLE);
351
352 error = devfs_get_cdevpriv((void**) &priv_user);
353 if (error != 0)
354 return (EINVAL);
355
356 /* Cleanup grefs and free pages. */
357 taskqueue_enqueue(taskqueue_thread, &cleanup_task);
358
359 /* Get file offset for this request. */
360 error = get_file_offset(priv_user, arg->count, &file_offset);
361 if (error != 0)
362 return (error);
363
364 /* Allocate grefs. */
365 grefs = malloc(sizeof(*grefs) * arg->count, M_GNTDEV, M_WAITOK);
366
367 for (i = 0; i < arg->count; i++) {
368 grefs[i].file_index = file_offset + i * PAGE_SIZE;
369 grefs[i].gref_id = GRANT_REF_INVALID;
370 grefs[i].notify = NULL;
371 grefs[i].page = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL
372 | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
373 if (grefs[i].page == NULL) {
374 log(LOG_ERR, "Page allocation failed.");
375 error = ENOMEM;
376 break;
377 }
378 if ((grefs[i].page->flags & PG_ZERO) == 0) {
379 /*
380 * Zero the allocated page, as we don't want to
381 * leak our memory to other domains.
382 */
383 pmap_zero_page(grefs[i].page);
384 }
385 grefs[i].page->valid = VM_PAGE_BITS_ALL;
386
387 error = gnttab_grant_foreign_access(arg->domid,
388 (VM_PAGE_TO_PHYS(grefs[i].page) >> PAGE_SHIFT),
389 readonly, &grefs[i].gref_id);
390 if (error != 0) {
391 log(LOG_ERR, "Grant Table Hypercall failed.");
392 break;
393 }
394 }
395
396 if (error != 0) {
397 /*
398 * If target domain maps the gref (by guessing the gref-id),
399 * then we can't clean it up yet and we have to leave the
400 * page in place so as to not leak our memory to that domain.
401 * Add it to a global list to be cleaned up later.
402 */
403 mtx_lock(&cleanup_data.to_kill_grefs_mtx);
404 for (i = 0; i < arg->count; i++)
405 STAILQ_INSERT_TAIL(&cleanup_data.to_kill_grefs,
406 &grefs[i], gref_next.list);
407 mtx_unlock(&cleanup_data.to_kill_grefs_mtx);
408
409 taskqueue_enqueue(taskqueue_thread, &cleanup_task);
410
411 return (error);
412 }
413
414 /* Copy the output values. */
415 arg->index = file_offset;
416 for (i = 0; i < arg->count; i++)
417 suword32(&arg->gref_ids[i], grefs[i].gref_id);
418
419 /* Modify the per user private data. */
420 mtx_lock(&priv_user->user_data_lock);
421 for (i = 0; i < arg->count; i++)
422 RB_INSERT(gref_tree_head, &priv_user->gref_tree, &grefs[i]);
423 mtx_unlock(&priv_user->user_data_lock);
424
425 return (error);
426 }
427
428 /*
429 * IOCTL_GNTDEV_DEALLOC_GREF
430 * Remove grant allocation information from the per user private data, so
431 * that it can't be mmapped anymore by the userspace program, and add it
432 * to the to-be-deleted grants global device-list.
433 */
434 static int
gntdev_dealloc_gref(struct ioctl_gntdev_dealloc_gref * arg)435 gntdev_dealloc_gref(struct ioctl_gntdev_dealloc_gref *arg)
436 {
437 int error;
438 uint32_t count;
439 struct gntdev_gref *gref, *gref_tmp;
440 struct per_user_data *priv_user;
441
442 error = devfs_get_cdevpriv((void**) &priv_user);
443 if (error != 0)
444 return (EINVAL);
445
446 gref = gntdev_find_grefs(priv_user, arg->index, arg->count);
447 if (gref == NULL) {
448 log(LOG_ERR, "Can't find requested grant-refs.");
449 return (EINVAL);
450 }
451
452 /* Remove the grefs from user private data. */
453 count = arg->count;
454 mtx_lock(&priv_user->user_data_lock);
455 mtx_lock(&cleanup_data.to_kill_grefs_mtx);
456 for (; gref != NULL && count > 0; gref = gref_tmp) {
457 gref_tmp = RB_NEXT(gref_tree_head, &priv_user->gref_tree, gref);
458 RB_REMOVE(gref_tree_head, &priv_user->gref_tree, gref);
459 STAILQ_INSERT_TAIL(&cleanup_data.to_kill_grefs, gref,
460 gref_next.list);
461 count--;
462 }
463 mtx_unlock(&cleanup_data.to_kill_grefs_mtx);
464 mtx_unlock(&priv_user->user_data_lock);
465
466 taskqueue_enqueue(taskqueue_thread, &cleanup_task);
467 put_file_offset(priv_user, arg->count, arg->index);
468
469 return (0);
470 }
471
472 /*-------------------- Grant Mapping Methods --------------------------------*/
473
474 struct gntdev_gmap_map {
475 vm_object_t mem;
476 struct resource *pseudo_phys_res;
477 int pseudo_phys_res_id;
478 vm_paddr_t phys_base_addr;
479 };
480
481 struct gntdev_gmap {
482 union gmap_next_union {
483 STAILQ_ENTRY(gntdev_gmap) list;
484 RB_ENTRY(gntdev_gmap) tree;
485 } gmap_next;
486 uint64_t file_index;
487 uint32_t count;
488 struct gnttab_map_grant_ref *grant_map_ops;
489 struct gntdev_gmap_map *map;
490 struct notify_data *notify;
491 };
492
493 static int
gmap_cmp(struct gntdev_gmap * g1,struct gntdev_gmap * g2)494 gmap_cmp(struct gntdev_gmap *g1, struct gntdev_gmap *g2)
495 {
496 return (g1->file_index - g2->file_index);
497 }
498
499 RB_GENERATE_STATIC(gmap_tree_head, gntdev_gmap, gmap_next.tree, gmap_cmp);
500
501 /*
502 * Traverse over the device-list of to-be-deleted grant mappings, and if
503 * the region is no longer mmapped by anyone, free the memory used to
504 * store information about the mapping.
505 */
506 static void
gmap_list_dtor(struct cleanup_data_struct * cleanup_data)507 gmap_list_dtor(struct cleanup_data_struct *cleanup_data)
508 {
509 struct gmap_list_head tmp_gmaps;
510 struct gntdev_gmap *gmap, *gmap_tmp, *gmap_previous;
511
512 STAILQ_INIT(&tmp_gmaps);
513 mtx_lock(&cleanup_data->to_kill_gmaps_mtx);
514 STAILQ_SWAP(&cleanup_data->to_kill_gmaps, &tmp_gmaps, gntdev_gmap);
515 mtx_unlock(&cleanup_data->to_kill_gmaps_mtx);
516
517 gmap_previous = NULL;
518 STAILQ_FOREACH_SAFE(gmap, &tmp_gmaps, gmap_next.list, gmap_tmp) {
519 if (gmap->map == NULL) {
520 if (gmap_previous == NULL)
521 STAILQ_REMOVE_HEAD(&tmp_gmaps, gmap_next.list);
522 else
523 STAILQ_REMOVE_AFTER(&tmp_gmaps, gmap_previous,
524 gmap_next.list);
525
526 if (gmap->notify)
527 free(gmap->notify, M_GNTDEV);
528 free(gmap->grant_map_ops, M_GNTDEV);
529 free(gmap, M_GNTDEV);
530 }
531 else
532 gmap_previous = gmap;
533 }
534
535 if (!STAILQ_EMPTY(&tmp_gmaps)) {
536 mtx_lock(&cleanup_data->to_kill_gmaps_mtx);
537 STAILQ_CONCAT(&cleanup_data->to_kill_gmaps, &tmp_gmaps);
538 mtx_unlock(&cleanup_data->to_kill_gmaps_mtx);
539 }
540 }
541
542 /*
543 * Find mapped grants for a given userspace program, by file-offset (index)
544 * and count, as supplied during the map-ioctl.
545 */
546 static struct gntdev_gmap*
gntdev_find_gmap(struct per_user_data * priv_user,uint64_t index,uint32_t count)547 gntdev_find_gmap(struct per_user_data *priv_user,
548 uint64_t index, uint32_t count)
549 {
550 struct gntdev_gmap find_gmap, *gmap;
551
552 find_gmap.file_index = index;
553
554 mtx_lock(&priv_user->user_data_lock);
555 gmap = RB_FIND(gmap_tree_head, &priv_user->gmap_tree, &find_gmap);
556 mtx_unlock(&priv_user->user_data_lock);
557
558 if (gmap != NULL && gmap->count == count)
559 return (gmap);
560 return (NULL);
561 }
562
563 /*
564 * Remove the pages from the mgtdevice pager, call the unmap hypercall,
565 * free the xenmem resource. This function is called during the
566 * destruction of the mgtdevice pager, which happens when all mmaps to
567 * it have been removed, and the unmap-ioctl has been performed.
568 */
569 static int
notify_unmap_cleanup(struct gntdev_gmap * gmap)570 notify_unmap_cleanup(struct gntdev_gmap *gmap)
571 {
572 uint32_t i;
573 int error, count;
574 vm_page_t m;
575 struct gnttab_unmap_grant_ref *unmap_ops;
576
577 unmap_ops = malloc(sizeof(struct gnttab_unmap_grant_ref) * gmap->count,
578 M_GNTDEV, M_WAITOK);
579
580 /* Enumerate freeable maps. */
581 count = 0;
582 for (i = 0; i < gmap->count; i++) {
583 if (gmap->grant_map_ops[i].handle != -1) {
584 unmap_ops[count].handle = gmap->grant_map_ops[i].handle;
585 unmap_ops[count].host_addr =
586 gmap->grant_map_ops[i].host_addr;
587 unmap_ops[count].dev_bus_addr = 0;
588 count++;
589 }
590 }
591
592 /* Perform notification. */
593 if (count > 0 && gmap->notify) {
594 vm_page_t page;
595 uint64_t page_offset;
596
597 page_offset = gmap->notify->index - gmap->file_index;
598 page = PHYS_TO_VM_PAGE(gmap->map->phys_base_addr + page_offset);
599 notify(gmap->notify, page);
600 }
601
602 /* Free the pages. */
603 VM_OBJECT_WLOCK(gmap->map->mem);
604 retry:
605 for (i = 0; i < gmap->count; i++) {
606 m = vm_page_lookup(gmap->map->mem, i);
607 if (m == NULL)
608 continue;
609 if (vm_page_sleep_if_busy(m, "pcmdum"))
610 goto retry;
611 cdev_pager_free_page(gmap->map->mem, m);
612 }
613 VM_OBJECT_WUNLOCK(gmap->map->mem);
614
615 /* Perform unmap hypercall. */
616 error = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref,
617 unmap_ops, count);
618
619 for (i = 0; i < gmap->count; i++) {
620 gmap->grant_map_ops[i].handle = -1;
621 gmap->grant_map_ops[i].host_addr = 0;
622 }
623
624 if (gmap->map) {
625 error = xenmem_free(gntdev_dev, gmap->map->pseudo_phys_res_id,
626 gmap->map->pseudo_phys_res);
627 KASSERT(error == 0,
628 ("Unable to release memory resource: %d", error));
629
630 free(gmap->map, M_GNTDEV);
631 gmap->map = NULL;
632 }
633
634 free(unmap_ops, M_GNTDEV);
635
636 return (error);
637 }
638
639 /*
640 * IOCTL_GNTDEV_MAP_GRANT_REF
641 * Populate structures for mapping the grant reference in the per user
642 * private data. Actual resource allocation and map hypercall is performed
643 * during the mmap.
644 */
645 static int
gntdev_map_grant_ref(struct ioctl_gntdev_map_grant_ref * arg)646 gntdev_map_grant_ref(struct ioctl_gntdev_map_grant_ref *arg)
647 {
648 uint32_t i;
649 int error;
650 struct gntdev_gmap *gmap;
651 struct per_user_data *priv_user;
652
653 error = devfs_get_cdevpriv((void**) &priv_user);
654 if (error != 0)
655 return (EINVAL);
656
657 gmap = malloc(sizeof(*gmap), M_GNTDEV, M_WAITOK | M_ZERO);
658 gmap->count = arg->count;
659 gmap->grant_map_ops =
660 malloc(sizeof(struct gnttab_map_grant_ref) * arg->count,
661 M_GNTDEV, M_WAITOK | M_ZERO);
662
663 for (i = 0; i < arg->count; i++) {
664 struct ioctl_gntdev_grant_ref ref;
665
666 error = copyin(&arg->refs[i], &ref, sizeof(ref));
667 if (error != 0) {
668 free(gmap->grant_map_ops, M_GNTDEV);
669 free(gmap, M_GNTDEV);
670 return (error);
671 }
672 gmap->grant_map_ops[i].dom = ref.domid;
673 gmap->grant_map_ops[i].ref = ref.ref;
674 gmap->grant_map_ops[i].handle = -1;
675 gmap->grant_map_ops[i].flags = GNTMAP_host_map;
676 }
677
678 error = get_file_offset(priv_user, arg->count, &gmap->file_index);
679 if (error != 0) {
680 free(gmap->grant_map_ops, M_GNTDEV);
681 free(gmap, M_GNTDEV);
682 return (error);
683 }
684
685 mtx_lock(&priv_user->user_data_lock);
686 RB_INSERT(gmap_tree_head, &priv_user->gmap_tree, gmap);
687 mtx_unlock(&priv_user->user_data_lock);
688
689 arg->index = gmap->file_index;
690
691 return (error);
692 }
693
694 /*
695 * IOCTL_GNTDEV_UNMAP_GRANT_REF
696 * Remove the map information from the per user private data and add it
697 * to the global device-list of mappings to be deleted. A reference to
698 * the mgtdevice pager is also decreased, the reason for which is
699 * explained in mmap_gmap().
700 */
701 static int
gntdev_unmap_grant_ref(struct ioctl_gntdev_unmap_grant_ref * arg)702 gntdev_unmap_grant_ref(struct ioctl_gntdev_unmap_grant_ref *arg)
703 {
704 int error;
705 struct gntdev_gmap *gmap;
706 struct per_user_data *priv_user;
707
708 error = devfs_get_cdevpriv((void**) &priv_user);
709 if (error != 0)
710 return (EINVAL);
711
712 gmap = gntdev_find_gmap(priv_user, arg->index, arg->count);
713 if (gmap == NULL) {
714 log(LOG_ERR, "Can't find requested grant-map.");
715 return (EINVAL);
716 }
717
718 mtx_lock(&priv_user->user_data_lock);
719 mtx_lock(&cleanup_data.to_kill_gmaps_mtx);
720 RB_REMOVE(gmap_tree_head, &priv_user->gmap_tree, gmap);
721 STAILQ_INSERT_TAIL(&cleanup_data.to_kill_gmaps, gmap, gmap_next.list);
722 mtx_unlock(&cleanup_data.to_kill_gmaps_mtx);
723 mtx_unlock(&priv_user->user_data_lock);
724
725 if (gmap->map)
726 vm_object_deallocate(gmap->map->mem);
727
728 taskqueue_enqueue(taskqueue_thread, &cleanup_task);
729 put_file_offset(priv_user, arg->count, arg->index);
730
731 return (0);
732 }
733
734 /*
735 * IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR
736 * Get file-offset and count for a given mapping, from the virtual address
737 * where the mapping is mmapped.
738 * Please note, this only works for grants mapped by this domain, and not
739 * grants allocated. Count doesn't make much sense in reference to grants
740 * allocated. Also, because this function is present in the linux gntdev
741 * device, but not in the linux gntalloc one, most userspace code only use
742 * it for mapped grants.
743 */
744 static int
gntdev_get_offset_for_vaddr(struct ioctl_gntdev_get_offset_for_vaddr * arg,struct thread * td)745 gntdev_get_offset_for_vaddr(struct ioctl_gntdev_get_offset_for_vaddr *arg,
746 struct thread *td)
747 {
748 int error;
749 vm_map_t map;
750 vm_map_entry_t entry;
751 vm_object_t mem;
752 vm_pindex_t pindex;
753 vm_prot_t prot;
754 boolean_t wired;
755 struct gntdev_gmap *gmap;
756 int rc;
757
758 map = &td->td_proc->p_vmspace->vm_map;
759 error = vm_map_lookup(&map, arg->vaddr, VM_PROT_NONE, &entry,
760 &mem, &pindex, &prot, &wired);
761 if (error != KERN_SUCCESS)
762 return (EINVAL);
763
764 if ((mem->type != OBJT_MGTDEVICE) ||
765 (mem->un_pager.devp.ops != &gntdev_gmap_pg_ops)) {
766 rc = EINVAL;
767 goto out;
768 }
769
770 gmap = mem->handle;
771 if (gmap == NULL ||
772 (entry->end - entry->start) != (gmap->count * PAGE_SIZE)) {
773 rc = EINVAL;
774 goto out;
775 }
776
777 arg->count = gmap->count;
778 arg->offset = gmap->file_index;
779 rc = 0;
780
781 out:
782 vm_map_lookup_done(map, entry);
783 return (rc);
784 }
785
786 /*-------------------- Grant Mapping Pager ----------------------------------*/
787
788 static int
gntdev_gmap_pg_ctor(void * handle,vm_ooffset_t size,vm_prot_t prot,vm_ooffset_t foff,struct ucred * cred,u_short * color)789 gntdev_gmap_pg_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
790 vm_ooffset_t foff, struct ucred *cred, u_short *color)
791 {
792
793 return (0);
794 }
795
796 static void
gntdev_gmap_pg_dtor(void * handle)797 gntdev_gmap_pg_dtor(void *handle)
798 {
799
800 notify_unmap_cleanup((struct gntdev_gmap *)handle);
801 }
802
803 static int
gntdev_gmap_pg_fault(vm_object_t object,vm_ooffset_t offset,int prot,vm_page_t * mres)804 gntdev_gmap_pg_fault(vm_object_t object, vm_ooffset_t offset, int prot,
805 vm_page_t *mres)
806 {
807 struct gntdev_gmap *gmap = object->handle;
808 vm_pindex_t pidx, ridx;
809 vm_page_t page, oldm;
810 vm_ooffset_t relative_offset;
811
812 if (gmap->map == NULL)
813 return (VM_PAGER_FAIL);
814
815 relative_offset = offset - gmap->file_index;
816
817 pidx = OFF_TO_IDX(offset);
818 ridx = OFF_TO_IDX(relative_offset);
819 if (ridx >= gmap->count ||
820 gmap->grant_map_ops[ridx].status != GNTST_okay)
821 return (VM_PAGER_FAIL);
822
823 page = PHYS_TO_VM_PAGE(gmap->map->phys_base_addr + relative_offset);
824 if (page == NULL)
825 return (VM_PAGER_FAIL);
826
827 KASSERT((page->flags & PG_FICTITIOUS) != 0,
828 ("not fictitious %p", page));
829 KASSERT(page->wire_count == 1, ("wire_count not 1 %p", page));
830 KASSERT(vm_page_busied(page) == 0, ("page %p is busy", page));
831
832 if (*mres != NULL) {
833 oldm = *mres;
834 vm_page_lock(oldm);
835 vm_page_free(oldm);
836 vm_page_unlock(oldm);
837 *mres = NULL;
838 }
839
840 vm_page_insert(page, object, pidx);
841 page->valid = VM_PAGE_BITS_ALL;
842 vm_page_xbusy(page);
843 *mres = page;
844 return (VM_PAGER_OK);
845 }
846
847 /*------------------ Grant Table Methods ------------------------------------*/
848
849 static void
notify(struct notify_data * notify,vm_page_t page)850 notify(struct notify_data *notify, vm_page_t page)
851 {
852 if (notify->action & UNMAP_NOTIFY_CLEAR_BYTE) {
853 uint8_t *mem;
854 uint64_t offset;
855
856 offset = notify->index & PAGE_MASK;
857 mem = (uint8_t *)pmap_quick_enter_page(page);
858 mem[offset] = 0;
859 pmap_quick_remove_page((vm_offset_t)mem);
860 }
861 if (notify->action & UNMAP_NOTIFY_SEND_EVENT) {
862 xen_intr_signal(notify->notify_evtchn_handle);
863 xen_intr_unbind(¬ify->notify_evtchn_handle);
864 }
865 notify->action = 0;
866 }
867
868 /*
869 * Helper to copy new arguments from the notify ioctl into
870 * the existing notify data.
871 */
872 static int
copy_notify_helper(struct notify_data * destination,struct ioctl_gntdev_unmap_notify * source)873 copy_notify_helper(struct notify_data *destination,
874 struct ioctl_gntdev_unmap_notify *source)
875 {
876 xen_intr_handle_t handlep = NULL;
877
878 /*
879 * "Get" before "Put"ting previous reference, as we might be
880 * holding the last reference to the event channel port.
881 */
882 if (source->action & UNMAP_NOTIFY_SEND_EVENT)
883 if (xen_intr_get_evtchn_from_port(source->event_channel_port,
884 &handlep) != 0)
885 return (EINVAL);
886
887 if (destination->action & UNMAP_NOTIFY_SEND_EVENT)
888 xen_intr_unbind(&destination->notify_evtchn_handle);
889
890 destination->action = source->action;
891 destination->event_channel_port = source->event_channel_port;
892 destination->index = source->index;
893 destination->notify_evtchn_handle = handlep;
894
895 return (0);
896 }
897
898 /*
899 * IOCTL_GNTDEV_SET_UNMAP_NOTIFY
900 * Set unmap notification inside the appropriate grant. It sends a
901 * notification when the grant is completely munmapped by this domain
902 * and ready for destruction.
903 */
904 static int
gntdev_set_unmap_notify(struct ioctl_gntdev_unmap_notify * arg)905 gntdev_set_unmap_notify(struct ioctl_gntdev_unmap_notify *arg)
906 {
907 int error;
908 uint64_t index;
909 struct per_user_data *priv_user;
910 struct gntdev_gref *gref = NULL;
911 struct gntdev_gmap *gmap;
912
913 error = devfs_get_cdevpriv((void**) &priv_user);
914 if (error != 0)
915 return (EINVAL);
916
917 if (arg->action & ~(UNMAP_NOTIFY_CLEAR_BYTE|UNMAP_NOTIFY_SEND_EVENT))
918 return (EINVAL);
919
920 index = arg->index & ~PAGE_MASK;
921 gref = gntdev_find_grefs(priv_user, index, 1);
922 if (gref) {
923 if (gref->notify == NULL)
924 gref->notify = malloc(sizeof(*arg), M_GNTDEV,
925 M_WAITOK | M_ZERO);
926 return (copy_notify_helper(gref->notify, arg));
927 }
928
929 error = EINVAL;
930 mtx_lock(&priv_user->user_data_lock);
931 RB_FOREACH(gmap, gmap_tree_head, &priv_user->gmap_tree) {
932 if (arg->index >= gmap->file_index &&
933 arg->index < gmap->file_index + gmap->count * PAGE_SIZE) {
934 if (gmap->notify == NULL)
935 gmap->notify = malloc(sizeof(*arg), M_GNTDEV,
936 M_WAITOK | M_ZERO);
937 error = copy_notify_helper(gmap->notify, arg);
938 break;
939 }
940 }
941 mtx_unlock(&priv_user->user_data_lock);
942
943 return (error);
944 }
945
946 /*------------------ Gntdev Char Device Methods -----------------------------*/
947
948 static void
cleanup_function(void * arg,__unused int pending)949 cleanup_function(void *arg, __unused int pending)
950 {
951
952 gref_list_dtor((struct cleanup_data_struct *) arg);
953 gmap_list_dtor((struct cleanup_data_struct *) arg);
954 }
955
956 static void
per_user_data_dtor(void * arg)957 per_user_data_dtor(void *arg)
958 {
959 struct gntdev_gref *gref, *gref_tmp;
960 struct gntdev_gmap *gmap, *gmap_tmp;
961 struct file_offset_struct *offset, *offset_tmp;
962 struct per_user_data *priv_user;
963
964 priv_user = (struct per_user_data *) arg;
965
966 mtx_lock(&priv_user->user_data_lock);
967
968 mtx_lock(&cleanup_data.to_kill_grefs_mtx);
969 RB_FOREACH_SAFE(gref, gref_tree_head, &priv_user->gref_tree, gref_tmp) {
970 RB_REMOVE(gref_tree_head, &priv_user->gref_tree, gref);
971 STAILQ_INSERT_TAIL(&cleanup_data.to_kill_grefs, gref,
972 gref_next.list);
973 }
974 mtx_unlock(&cleanup_data.to_kill_grefs_mtx);
975
976 mtx_lock(&cleanup_data.to_kill_gmaps_mtx);
977 RB_FOREACH_SAFE(gmap, gmap_tree_head, &priv_user->gmap_tree, gmap_tmp) {
978 RB_REMOVE(gmap_tree_head, &priv_user->gmap_tree, gmap);
979 STAILQ_INSERT_TAIL(&cleanup_data.to_kill_gmaps, gmap,
980 gmap_next.list);
981 if (gmap->map)
982 vm_object_deallocate(gmap->map->mem);
983 }
984 mtx_unlock(&cleanup_data.to_kill_gmaps_mtx);
985
986 RB_FOREACH_SAFE(offset, file_offset_head, &priv_user->file_offset,
987 offset_tmp) {
988 RB_REMOVE(file_offset_head, &priv_user->file_offset, offset);
989 free(offset, M_GNTDEV);
990 }
991
992 mtx_unlock(&priv_user->user_data_lock);
993
994 taskqueue_enqueue(taskqueue_thread, &cleanup_task);
995
996 mtx_destroy(&priv_user->user_data_lock);
997 free(priv_user, M_GNTDEV);
998 }
999
1000 static int
gntdev_open(struct cdev * dev,int flag,int otyp,struct thread * td)1001 gntdev_open(struct cdev *dev, int flag, int otyp, struct thread *td)
1002 {
1003 int error;
1004 struct per_user_data *priv_user;
1005 struct file_offset_struct *offset;
1006
1007 priv_user = malloc(sizeof(*priv_user), M_GNTDEV, M_WAITOK | M_ZERO);
1008 RB_INIT(&priv_user->gref_tree);
1009 RB_INIT(&priv_user->gmap_tree);
1010 RB_INIT(&priv_user->file_offset);
1011 offset = malloc(sizeof(*offset), M_GNTDEV, M_WAITOK | M_ZERO);
1012 offset->file_offset = 0;
1013 offset->count = MAX_OFFSET_COUNT;
1014 RB_INSERT(file_offset_head, &priv_user->file_offset, offset);
1015 mtx_init(&priv_user->user_data_lock,
1016 "per user data mutex", NULL, MTX_DEF);
1017
1018 error = devfs_set_cdevpriv(priv_user, per_user_data_dtor);
1019 if (error != 0)
1020 per_user_data_dtor(priv_user);
1021
1022 return (error);
1023 }
1024
1025 static int
gntdev_ioctl(struct cdev * dev,u_long cmd,caddr_t data,int fflag,struct thread * td)1026 gntdev_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
1027 int fflag, struct thread *td)
1028 {
1029 int error;
1030
1031 switch (cmd) {
1032 case IOCTL_GNTDEV_SET_UNMAP_NOTIFY:
1033 error = gntdev_set_unmap_notify(
1034 (struct ioctl_gntdev_unmap_notify*) data);
1035 break;
1036 case IOCTL_GNTDEV_ALLOC_GREF:
1037 error = gntdev_alloc_gref(
1038 (struct ioctl_gntdev_alloc_gref*) data);
1039 break;
1040 case IOCTL_GNTDEV_DEALLOC_GREF:
1041 error = gntdev_dealloc_gref(
1042 (struct ioctl_gntdev_dealloc_gref*) data);
1043 break;
1044 case IOCTL_GNTDEV_MAP_GRANT_REF:
1045 error = gntdev_map_grant_ref(
1046 (struct ioctl_gntdev_map_grant_ref*) data);
1047 break;
1048 case IOCTL_GNTDEV_UNMAP_GRANT_REF:
1049 error = gntdev_unmap_grant_ref(
1050 (struct ioctl_gntdev_unmap_grant_ref*) data);
1051 break;
1052 case IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR:
1053 error = gntdev_get_offset_for_vaddr(
1054 (struct ioctl_gntdev_get_offset_for_vaddr*) data, td);
1055 break;
1056 default:
1057 error = ENOSYS;
1058 break;
1059 }
1060
1061 return (error);
1062 }
1063
1064 /*
1065 * MMAP an allocated grant into user memory.
1066 * Please note, that the grants must not already be mmapped, otherwise
1067 * this function will fail.
1068 */
1069 static int
mmap_gref(struct per_user_data * priv_user,struct gntdev_gref * gref_start,uint32_t count,vm_size_t size,struct vm_object ** object)1070 mmap_gref(struct per_user_data *priv_user, struct gntdev_gref *gref_start,
1071 uint32_t count, vm_size_t size, struct vm_object **object)
1072 {
1073 vm_object_t mem_obj;
1074 struct gntdev_gref *gref;
1075
1076 mem_obj = vm_object_allocate(OBJT_PHYS, size);
1077 if (mem_obj == NULL)
1078 return (ENOMEM);
1079
1080 mtx_lock(&priv_user->user_data_lock);
1081 VM_OBJECT_WLOCK(mem_obj);
1082 for (gref = gref_start; gref != NULL && count > 0; gref =
1083 RB_NEXT(gref_tree_head, &priv_user->gref_tree, gref)) {
1084 if (gref->page->object)
1085 break;
1086
1087 vm_page_insert(gref->page, mem_obj,
1088 OFF_TO_IDX(gref->file_index));
1089
1090 count--;
1091 }
1092 VM_OBJECT_WUNLOCK(mem_obj);
1093 mtx_unlock(&priv_user->user_data_lock);
1094
1095 if (count) {
1096 vm_object_deallocate(mem_obj);
1097 return (EINVAL);
1098 }
1099
1100 *object = mem_obj;
1101
1102 return (0);
1103
1104 }
1105
1106 /*
1107 * MMAP a mapped grant into user memory.
1108 */
1109 static int
mmap_gmap(struct per_user_data * priv_user,struct gntdev_gmap * gmap_start,vm_ooffset_t * offset,vm_size_t size,struct vm_object ** object,int nprot)1110 mmap_gmap(struct per_user_data *priv_user, struct gntdev_gmap *gmap_start,
1111 vm_ooffset_t *offset, vm_size_t size, struct vm_object **object, int nprot)
1112 {
1113 uint32_t i;
1114 int error;
1115
1116 /*
1117 * The grant map hypercall might already be done.
1118 * If that is the case, increase a reference to the
1119 * vm object and return the already allocated object.
1120 */
1121 if (gmap_start->map) {
1122 vm_object_reference(gmap_start->map->mem);
1123 *object = gmap_start->map->mem;
1124 return (0);
1125 }
1126
1127 gmap_start->map = malloc(sizeof(*(gmap_start->map)), M_GNTDEV,
1128 M_WAITOK | M_ZERO);
1129
1130 /* Allocate the xen pseudo physical memory resource. */
1131 gmap_start->map->pseudo_phys_res_id = 0;
1132 gmap_start->map->pseudo_phys_res = xenmem_alloc(gntdev_dev,
1133 &gmap_start->map->pseudo_phys_res_id, size);
1134 if (gmap_start->map->pseudo_phys_res == NULL) {
1135 free(gmap_start->map, M_GNTDEV);
1136 gmap_start->map = NULL;
1137 return (ENOMEM);
1138 }
1139 gmap_start->map->phys_base_addr =
1140 rman_get_start(gmap_start->map->pseudo_phys_res);
1141
1142 /* Allocate the mgtdevice pager. */
1143 gmap_start->map->mem = cdev_pager_allocate(gmap_start, OBJT_MGTDEVICE,
1144 &gntdev_gmap_pg_ops, size, nprot, *offset, NULL);
1145 if (gmap_start->map->mem == NULL) {
1146 xenmem_free(gntdev_dev, gmap_start->map->pseudo_phys_res_id,
1147 gmap_start->map->pseudo_phys_res);
1148 free(gmap_start->map, M_GNTDEV);
1149 gmap_start->map = NULL;
1150 return (ENOMEM);
1151 }
1152
1153 for (i = 0; i < gmap_start->count; i++) {
1154 gmap_start->grant_map_ops[i].host_addr =
1155 gmap_start->map->phys_base_addr + i * PAGE_SIZE;
1156
1157 if ((nprot & PROT_WRITE) == 0)
1158 gmap_start->grant_map_ops[i].flags |= GNTMAP_readonly;
1159 }
1160 /* Make the MAP hypercall. */
1161 error = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref,
1162 gmap_start->grant_map_ops, gmap_start->count);
1163 if (error != 0) {
1164 /*
1165 * Deallocate pager.
1166 * Pager deallocation will automatically take care of
1167 * xenmem deallocation, etc.
1168 */
1169 vm_object_deallocate(gmap_start->map->mem);
1170
1171 return (EINVAL);
1172 }
1173
1174 /* Retry EAGAIN maps. */
1175 for (i = 0; i < gmap_start->count; i++) {
1176 int delay = 1;
1177 while (delay < 256 &&
1178 gmap_start->grant_map_ops[i].status == GNTST_eagain) {
1179 HYPERVISOR_grant_table_op( GNTTABOP_map_grant_ref,
1180 &gmap_start->grant_map_ops[i], 1);
1181 pause(("gntmap"), delay * SBT_1MS);
1182 delay++;
1183 }
1184 if (gmap_start->grant_map_ops[i].status == GNTST_eagain)
1185 gmap_start->grant_map_ops[i].status = GNTST_bad_page;
1186
1187 if (gmap_start->grant_map_ops[i].status != GNTST_okay) {
1188 /*
1189 * Deallocate pager.
1190 * Pager deallocation will automatically take care of
1191 * xenmem deallocation, notification, unmap hypercall,
1192 * etc.
1193 */
1194 vm_object_deallocate(gmap_start->map->mem);
1195
1196 return (EINVAL);
1197 }
1198 }
1199
1200 /*
1201 * Add a reference to the vm object. We do not want
1202 * the vm object to be deleted when all the mmaps are
1203 * unmapped, because it may be re-mmapped. Instead,
1204 * we want the object to be deleted, when along with
1205 * munmaps, we have also processed the unmap-ioctl.
1206 */
1207 vm_object_reference(gmap_start->map->mem);
1208
1209 *object = gmap_start->map->mem;
1210
1211 return (0);
1212 }
1213
1214 static int
gntdev_mmap_single(struct cdev * cdev,vm_ooffset_t * offset,vm_size_t size,struct vm_object ** object,int nprot)1215 gntdev_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t size,
1216 struct vm_object **object, int nprot)
1217 {
1218 int error;
1219 uint32_t count;
1220 struct gntdev_gref *gref_start;
1221 struct gntdev_gmap *gmap_start;
1222 struct per_user_data *priv_user;
1223
1224 error = devfs_get_cdevpriv((void**) &priv_user);
1225 if (error != 0)
1226 return (EINVAL);
1227
1228 count = OFF_TO_IDX(size);
1229
1230 gref_start = gntdev_find_grefs(priv_user, *offset, count);
1231 if (gref_start) {
1232 error = mmap_gref(priv_user, gref_start, count, size, object);
1233 return (error);
1234 }
1235
1236 gmap_start = gntdev_find_gmap(priv_user, *offset, count);
1237 if (gmap_start) {
1238 error = mmap_gmap(priv_user, gmap_start, offset, size, object,
1239 nprot);
1240 return (error);
1241 }
1242
1243 return (EINVAL);
1244 }
1245
1246 /*------------------ Private Device Attachment Functions --------------------*/
1247 static void
gntdev_identify(driver_t * driver,device_t parent)1248 gntdev_identify(driver_t *driver, device_t parent)
1249 {
1250
1251 KASSERT((xen_domain()),
1252 ("Trying to attach gntdev device on non Xen domain"));
1253
1254 if (BUS_ADD_CHILD(parent, 0, "gntdev", 0) == NULL)
1255 panic("unable to attach gntdev user-space device");
1256 }
1257
1258 static int
gntdev_probe(device_t dev)1259 gntdev_probe(device_t dev)
1260 {
1261
1262 gntdev_dev = dev;
1263 device_set_desc(dev, "Xen grant-table user-space device");
1264 return (BUS_PROBE_NOWILDCARD);
1265 }
1266
1267 static int
gntdev_attach(device_t dev)1268 gntdev_attach(device_t dev)
1269 {
1270
1271 make_dev_credf(MAKEDEV_ETERNAL, &gntdev_devsw, 0, NULL, UID_ROOT,
1272 GID_WHEEL, 0600, "xen/gntdev");
1273 return (0);
1274 }
1275
1276 /*-------------------- Private Device Attachment Data -----------------------*/
1277 static device_method_t gntdev_methods[] = {
1278 DEVMETHOD(device_identify, gntdev_identify),
1279 DEVMETHOD(device_probe, gntdev_probe),
1280 DEVMETHOD(device_attach, gntdev_attach),
1281 DEVMETHOD_END
1282 };
1283
1284 static driver_t gntdev_driver = {
1285 "gntdev",
1286 gntdev_methods,
1287 0,
1288 };
1289
1290 devclass_t gntdev_devclass;
1291
1292 DRIVER_MODULE(gntdev, xenpv, gntdev_driver, gntdev_devclass, 0, 0);
1293 MODULE_DEPEND(gntdev, xenpv, 1, 1, 1);
1294