1 /*-
2 * Copyright (c) 2018 VMware, Inc.
3 *
4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
5 */
6
7 /* VMCI QueuePair API implementation. */
8
9 #include <sys/cdefs.h>
10 __FBSDID("$FreeBSD$");
11
12 #include "vmci.h"
13 #include "vmci_driver.h"
14 #include "vmci_event.h"
15 #include "vmci_kernel_api.h"
16 #include "vmci_kernel_defs.h"
17 #include "vmci_queue_pair.h"
18
19 #define LGPFX "vmci_queue_pair: "
20
21 struct queue_pair_entry {
22 vmci_list_item(queue_pair_entry) list_item;
23 struct vmci_handle handle;
24 vmci_id peer;
25 uint32_t flags;
26 uint64_t produce_size;
27 uint64_t consume_size;
28 uint32_t ref_count;
29 };
30
31 struct qp_guest_endpoint {
32 struct queue_pair_entry qp;
33 uint64_t num_ppns;
34 void *produce_q;
35 void *consume_q;
36 bool hibernate_failure;
37 struct ppn_set ppn_set;
38 };
39
40 struct queue_pair_list {
41 vmci_list(queue_pair_entry) head;
42 volatile int hibernate;
43 vmci_mutex mutex;
44 };
45
46 #define QPE_NUM_PAGES(_QPE) \
47 ((uint32_t)(CEILING(_QPE.produce_size, PAGE_SIZE) + \
48 CEILING(_QPE.consume_size, PAGE_SIZE) + 2))
49
50 static struct queue_pair_list qp_guest_endpoints;
51
52 static struct queue_pair_entry *queue_pair_list_find_entry(
53 struct queue_pair_list *qp_list, struct vmci_handle handle);
54 static void queue_pair_list_add_entry(struct queue_pair_list *qp_list,
55 struct queue_pair_entry *entry);
56 static void queue_pair_list_remove_entry(struct queue_pair_list *qp_list,
57 struct queue_pair_entry *entry);
58 static struct queue_pair_entry *queue_pair_list_get_head(
59 struct queue_pair_list *qp_list);
60 static int queue_pair_notify_peer_local(bool attach,
61 struct vmci_handle handle);
62 static struct qp_guest_endpoint *qp_guest_endpoint_create(
63 struct vmci_handle handle, vmci_id peer, uint32_t flags,
64 uint64_t produce_size, uint64_t consume_size,
65 void *produce_q, void *consume_q);
66 static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry);
67 static int vmci_queue_pair_alloc_hypercall(
68 const struct qp_guest_endpoint *entry);
69 static int vmci_queue_pair_alloc_guest_work(struct vmci_handle *handle,
70 struct vmci_queue **produce_q, uint64_t produce_size,
71 struct vmci_queue **consume_q, uint64_t consume_size,
72 vmci_id peer, uint32_t flags,
73 vmci_privilege_flags priv_flags);
74 static int vmci_queue_pair_detach_guest_work(struct vmci_handle handle);
75 static int vmci_queue_pair_detach_hypercall(struct vmci_handle handle);
76
77 /*
78 *------------------------------------------------------------------------------
79 *
80 * vmci_queue_pair_alloc --
81 *
82 * Allocates a VMCI QueuePair. Only checks validity of input arguments. The
83 * real work is done in the host or guest specific function.
84 *
85 * Results:
86 * VMCI_SUCCESS on success, appropriate error code otherwise.
87 *
88 * Side effects:
89 * None.
90 *
91 *------------------------------------------------------------------------------
92 */
93
94 int
vmci_queue_pair_alloc(struct vmci_handle * handle,struct vmci_queue ** produce_q,uint64_t produce_size,struct vmci_queue ** consume_q,uint64_t consume_size,vmci_id peer,uint32_t flags,vmci_privilege_flags priv_flags)95 vmci_queue_pair_alloc(struct vmci_handle *handle, struct vmci_queue **produce_q,
96 uint64_t produce_size, struct vmci_queue **consume_q, uint64_t consume_size,
97 vmci_id peer, uint32_t flags, vmci_privilege_flags priv_flags)
98 {
99
100 if (!handle || !produce_q || !consume_q ||
101 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
102 return (VMCI_ERROR_INVALID_ARGS);
103
104 return (vmci_queue_pair_alloc_guest_work(handle, produce_q,
105 produce_size, consume_q, consume_size, peer, flags, priv_flags));
106 }
107
108 /*
109 *------------------------------------------------------------------------------
110 *
111 * vmci_queue_pair_detach --
112 *
113 * Detaches from a VMCI QueuePair. Only checks validity of input argument.
114 * Real work is done in the host or guest specific function.
115 *
116 * Results:
117 * Success or failure.
118 *
119 * Side effects:
120 * Memory is freed.
121 *
122 *------------------------------------------------------------------------------
123 */
124
125 int
vmci_queue_pair_detach(struct vmci_handle handle)126 vmci_queue_pair_detach(struct vmci_handle handle)
127 {
128
129 if (VMCI_HANDLE_INVALID(handle))
130 return (VMCI_ERROR_INVALID_ARGS);
131
132 return (vmci_queue_pair_detach_guest_work(handle));
133 }
134
135 /*
136 *------------------------------------------------------------------------------
137 *
138 * queue_pair_list_init --
139 *
140 * Initializes the list of QueuePairs.
141 *
142 * Results:
143 * Success or failure.
144 *
145 * Side effects:
146 * None.
147 *
148 *------------------------------------------------------------------------------
149 */
150
151 static inline int
queue_pair_list_init(struct queue_pair_list * qp_list)152 queue_pair_list_init(struct queue_pair_list *qp_list)
153 {
154 int ret;
155
156 vmci_list_init(&qp_list->head);
157 atomic_store_int(&qp_list->hibernate, 0);
158 ret = vmci_mutex_init(&qp_list->mutex, "VMCI QP List lock");
159 return (ret);
160 }
161
162 /*
163 *------------------------------------------------------------------------------
164 *
165 * queue_pair_list_destroy --
166 *
167 * Destroy the list's mutex.
168 *
169 * Results:
170 * None.
171 *
172 * Side effects:
173 * None.
174 *
175 *------------------------------------------------------------------------------
176 */
177
178 static inline void
queue_pair_list_destroy(struct queue_pair_list * qp_list)179 queue_pair_list_destroy(struct queue_pair_list *qp_list)
180 {
181
182 vmci_mutex_destroy(&qp_list->mutex);
183 vmci_list_init(&qp_list->head);
184 }
185
186 /*
187 *------------------------------------------------------------------------------
188 *
189 * queue_pair_list_find_entry --
190 *
191 * Finds the entry in the list corresponding to a given handle. Assumes that
192 * the list is locked.
193 *
194 * Results:
195 * Pointer to entry.
196 *
197 * Side effects:
198 * None.
199 *
200 *------------------------------------------------------------------------------
201 */
202
203 static struct queue_pair_entry *
queue_pair_list_find_entry(struct queue_pair_list * qp_list,struct vmci_handle handle)204 queue_pair_list_find_entry(struct queue_pair_list *qp_list,
205 struct vmci_handle handle)
206 {
207 struct queue_pair_entry *next;
208
209 if (VMCI_HANDLE_INVALID(handle))
210 return (NULL);
211
212 vmci_list_scan(next, &qp_list->head, list_item) {
213 if (VMCI_HANDLE_EQUAL(next->handle, handle))
214 return (next);
215 }
216
217 return (NULL);
218 }
219
220 /*
221 *------------------------------------------------------------------------------
222 *
223 * queue_pair_list_add_entry --
224 *
225 * Adds the given entry to the list. Assumes that the list is locked.
226 *
227 * Results:
228 * None.
229 *
230 * Side effects:
231 * None.
232 *
233 *------------------------------------------------------------------------------
234 */
235
236 static void
queue_pair_list_add_entry(struct queue_pair_list * qp_list,struct queue_pair_entry * entry)237 queue_pair_list_add_entry(struct queue_pair_list *qp_list,
238 struct queue_pair_entry *entry)
239 {
240
241 if (entry)
242 vmci_list_insert(&qp_list->head, entry, list_item);
243 }
244
245 /*
246 *------------------------------------------------------------------------------
247 *
248 * queue_pair_list_remove_entry --
249 *
250 * Removes the given entry from the list. Assumes that the list is locked.
251 *
252 * Results:
253 * None.
254 *
255 * Side effects:
256 * None.
257 *
258 *------------------------------------------------------------------------------
259 */
260
261 static void
queue_pair_list_remove_entry(struct queue_pair_list * qp_list,struct queue_pair_entry * entry)262 queue_pair_list_remove_entry(struct queue_pair_list *qp_list,
263 struct queue_pair_entry *entry)
264 {
265
266 if (entry)
267 vmci_list_remove(entry, list_item);
268 }
269
270 /*
271 *------------------------------------------------------------------------------
272 *
273 * queue_pair_list_get_head --
274 *
275 * Returns the entry from the head of the list. Assumes that the list is
276 * locked.
277 *
278 * Results:
279 * Pointer to entry.
280 *
281 * Side effects:
282 * None.
283 *
284 *------------------------------------------------------------------------------
285 */
286
287 static struct queue_pair_entry *
queue_pair_list_get_head(struct queue_pair_list * qp_list)288 queue_pair_list_get_head(struct queue_pair_list *qp_list)
289 {
290
291 return (vmci_list_first(&qp_list->head));
292 }
293
294 /*
295 *------------------------------------------------------------------------------
296 *
297 * vmci_qp_guest_endpoints_init --
298 *
299 * Initalizes data structure state keeping track of queue pair guest
300 * endpoints.
301 *
302 * Results:
303 * VMCI_SUCCESS on success and appropriate failure code otherwise.
304 *
305 * Side effects:
306 * None.
307 *
308 *------------------------------------------------------------------------------
309 */
310
311 int
vmci_qp_guest_endpoints_init(void)312 vmci_qp_guest_endpoints_init(void)
313 {
314
315 return (queue_pair_list_init(&qp_guest_endpoints));
316 }
317
318 /*
319 *------------------------------------------------------------------------------
320 *
321 * vmci_qp_guest_endpoints_exit --
322 *
323 * Destroys all guest queue pair endpoints. If active guest queue pairs
324 * still exist, hypercalls to attempt detach from these queue pairs will be
325 * made. Any failure to detach is silently ignored.
326 *
327 * Results:
328 * None.
329 *
330 * Side effects:
331 * None.
332 *
333 *------------------------------------------------------------------------------
334 */
335
336 void
vmci_qp_guest_endpoints_exit(void)337 vmci_qp_guest_endpoints_exit(void)
338 {
339 struct qp_guest_endpoint *entry;
340
341 vmci_mutex_acquire(&qp_guest_endpoints.mutex);
342
343 while ((entry =
344 (struct qp_guest_endpoint *)queue_pair_list_get_head(
345 &qp_guest_endpoints)) != NULL) {
346 /*
347 * Don't make a hypercall for local QueuePairs.
348 */
349 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL))
350 vmci_queue_pair_detach_hypercall(entry->qp.handle);
351 /*
352 * We cannot fail the exit, so let's reset ref_count.
353 */
354 entry->qp.ref_count = 0;
355 queue_pair_list_remove_entry(&qp_guest_endpoints, &entry->qp);
356 qp_guest_endpoint_destroy(entry);
357 }
358
359 atomic_store_int(&qp_guest_endpoints.hibernate, 0);
360 vmci_mutex_release(&qp_guest_endpoints.mutex);
361 queue_pair_list_destroy(&qp_guest_endpoints);
362 }
363
364 /*
365 *------------------------------------------------------------------------------
366 *
367 * vmci_qp_guest_endpoints_sync --
368 *
369 * Use this as a synchronization point when setting globals, for example,
370 * during device shutdown.
371 *
372 * Results:
373 * true.
374 *
375 * Side effects:
376 * None.
377 *
378 *------------------------------------------------------------------------------
379 */
380
381 void
vmci_qp_guest_endpoints_sync(void)382 vmci_qp_guest_endpoints_sync(void)
383 {
384
385 vmci_mutex_acquire(&qp_guest_endpoints.mutex);
386 vmci_mutex_release(&qp_guest_endpoints.mutex);
387 }
388
389 /*
390 *------------------------------------------------------------------------------
391 *
392 * qp_guest_endpoint_create --
393 *
394 * Allocates and initializes a qp_guest_endpoint structure. Allocates a
395 * QueuePair rid (and handle) iff the given entry has an invalid handle.
396 * 0 through VMCI_RESERVED_RESOURCE_ID_MAX are reserved handles. Assumes
397 * that the QP list mutex is held by the caller.
398 *
399 * Results:
400 * Pointer to structure intialized.
401 *
402 * Side effects:
403 * None.
404 *
405 *------------------------------------------------------------------------------
406 */
407
408 struct qp_guest_endpoint *
qp_guest_endpoint_create(struct vmci_handle handle,vmci_id peer,uint32_t flags,uint64_t produce_size,uint64_t consume_size,void * produce_q,void * consume_q)409 qp_guest_endpoint_create(struct vmci_handle handle, vmci_id peer,
410 uint32_t flags, uint64_t produce_size, uint64_t consume_size,
411 void *produce_q, void *consume_q)
412 {
413 struct qp_guest_endpoint *entry;
414 static vmci_id queue_pair_rid;
415 const uint64_t num_ppns = CEILING(produce_size, PAGE_SIZE) +
416 CEILING(consume_size, PAGE_SIZE) +
417 2; /* One page each for the queue headers. */
418
419 queue_pair_rid = VMCI_RESERVED_RESOURCE_ID_MAX + 1;
420
421 ASSERT((produce_size || consume_size) && produce_q && consume_q);
422
423 if (VMCI_HANDLE_INVALID(handle)) {
424 vmci_id context_id = vmci_get_context_id();
425 vmci_id old_rid = queue_pair_rid;
426
427 /*
428 * Generate a unique QueuePair rid. Keep on trying until we
429 * wrap around in the RID space.
430 */
431 ASSERT(old_rid > VMCI_RESERVED_RESOURCE_ID_MAX);
432 do {
433 handle = VMCI_MAKE_HANDLE(context_id, queue_pair_rid);
434 entry =
435 (struct qp_guest_endpoint *)
436 queue_pair_list_find_entry(&qp_guest_endpoints,
437 handle);
438 queue_pair_rid++;
439 if (UNLIKELY(!queue_pair_rid)) {
440 /*
441 * Skip the reserved rids.
442 */
443 queue_pair_rid =
444 VMCI_RESERVED_RESOURCE_ID_MAX + 1;
445 }
446 } while (entry && queue_pair_rid != old_rid);
447
448 if (UNLIKELY(entry != NULL)) {
449 ASSERT(queue_pair_rid == old_rid);
450 /*
451 * We wrapped around --- no rids were free.
452 */
453 return (NULL);
454 }
455 }
456
457 ASSERT(!VMCI_HANDLE_INVALID(handle) &&
458 queue_pair_list_find_entry(&qp_guest_endpoints, handle) == NULL);
459 entry = vmci_alloc_kernel_mem(sizeof(*entry), VMCI_MEMORY_NORMAL);
460 if (entry) {
461 entry->qp.handle = handle;
462 entry->qp.peer = peer;
463 entry->qp.flags = flags;
464 entry->qp.produce_size = produce_size;
465 entry->qp.consume_size = consume_size;
466 entry->qp.ref_count = 0;
467 entry->num_ppns = num_ppns;
468 memset(&entry->ppn_set, 0, sizeof(entry->ppn_set));
469 entry->produce_q = produce_q;
470 entry->consume_q = consume_q;
471 }
472 return (entry);
473 }
474
475 /*
476 *------------------------------------------------------------------------------
477 *
478 * qp_guest_endpoint_destroy --
479 *
480 * Frees a qp_guest_endpoint structure.
481 *
482 * Results:
483 * None.
484 *
485 * Side effects:
486 * None.
487 *
488 *------------------------------------------------------------------------------
489 */
490
491 void
qp_guest_endpoint_destroy(struct qp_guest_endpoint * entry)492 qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
493 {
494
495 ASSERT(entry);
496 ASSERT(entry->qp.ref_count == 0);
497
498 vmci_free_ppn_set(&entry->ppn_set);
499 vmci_free_queue(entry->produce_q, entry->qp.produce_size);
500 vmci_free_queue(entry->consume_q, entry->qp.consume_size);
501 vmci_free_kernel_mem(entry, sizeof(*entry));
502 }
503
504 /*
505 *------------------------------------------------------------------------------
506 *
507 * vmci_queue_pair_alloc_hypercall --
508 *
509 * Helper to make a QueuePairAlloc hypercall when the driver is
510 * supporting a guest device.
511 *
512 * Results:
513 * Result of the hypercall.
514 *
515 * Side effects:
516 * Memory is allocated & freed.
517 *
518 *------------------------------------------------------------------------------
519 */
520 static int
vmci_queue_pair_alloc_hypercall(const struct qp_guest_endpoint * entry)521 vmci_queue_pair_alloc_hypercall(const struct qp_guest_endpoint *entry)
522 {
523 struct vmci_queue_pair_alloc_msg *alloc_msg;
524 size_t msg_size;
525 int result;
526
527 if (!entry || entry->num_ppns <= 2)
528 return (VMCI_ERROR_INVALID_ARGS);
529
530 ASSERT(!(entry->qp.flags & VMCI_QPFLAG_LOCAL));
531
532 msg_size = sizeof(*alloc_msg) + (size_t)entry->num_ppns * sizeof(PPN);
533 alloc_msg = vmci_alloc_kernel_mem(msg_size, VMCI_MEMORY_NORMAL);
534 if (!alloc_msg)
535 return (VMCI_ERROR_NO_MEM);
536
537 alloc_msg->hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
538 VMCI_QUEUEPAIR_ALLOC);
539 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
540 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
541 alloc_msg->handle = entry->qp.handle;
542 alloc_msg->peer = entry->qp.peer;
543 alloc_msg->flags = entry->qp.flags;
544 alloc_msg->produce_size = entry->qp.produce_size;
545 alloc_msg->consume_size = entry->qp.consume_size;
546 alloc_msg->num_ppns = entry->num_ppns;
547 result = vmci_populate_ppn_list((uint8_t *)alloc_msg +
548 sizeof(*alloc_msg), &entry->ppn_set);
549 if (result == VMCI_SUCCESS)
550 result = vmci_send_datagram((struct vmci_datagram *)alloc_msg);
551 vmci_free_kernel_mem(alloc_msg, msg_size);
552
553 return (result);
554 }
555
556 /*
557 *------------------------------------------------------------------------------
558 *
559 * vmci_queue_pair_alloc_guest_work --
560 *
561 * This functions handles the actual allocation of a VMCI queue pair guest
562 * endpoint. Allocates physical pages for the queue pair. It makes OS
563 * dependent calls through generic wrappers.
564 *
565 * Results:
566 * Success or failure.
567 *
568 * Side effects:
569 * Memory is allocated.
570 *
571 *------------------------------------------------------------------------------
572 */
573
574 static int
vmci_queue_pair_alloc_guest_work(struct vmci_handle * handle,struct vmci_queue ** produce_q,uint64_t produce_size,struct vmci_queue ** consume_q,uint64_t consume_size,vmci_id peer,uint32_t flags,vmci_privilege_flags priv_flags)575 vmci_queue_pair_alloc_guest_work(struct vmci_handle *handle,
576 struct vmci_queue **produce_q, uint64_t produce_size,
577 struct vmci_queue **consume_q, uint64_t consume_size, vmci_id peer,
578 uint32_t flags, vmci_privilege_flags priv_flags)
579 {
580 struct qp_guest_endpoint *queue_pair_entry = NULL;
581 void *my_consume_q = NULL;
582 void *my_produce_q = NULL;
583 const uint64_t num_consume_pages = CEILING(consume_size, PAGE_SIZE) + 1;
584 const uint64_t num_produce_pages = CEILING(produce_size, PAGE_SIZE) + 1;
585 int result;
586
587 ASSERT(handle && produce_q && consume_q &&
588 (produce_size || consume_size));
589
590 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
591 return (VMCI_ERROR_NO_ACCESS);
592
593 vmci_mutex_acquire(&qp_guest_endpoints.mutex);
594
595 if ((atomic_load_int(&qp_guest_endpoints.hibernate) == 1) &&
596 !(flags & VMCI_QPFLAG_LOCAL)) {
597 /*
598 * While guest OS is in hibernate state, creating non-local
599 * queue pairs is not allowed after the point where the VMCI
600 * guest driver converted the existing queue pairs to local
601 * ones.
602 */
603
604 result = VMCI_ERROR_UNAVAILABLE;
605 goto error;
606 }
607
608 if ((queue_pair_entry =
609 (struct qp_guest_endpoint *)queue_pair_list_find_entry(
610 &qp_guest_endpoints, *handle)) != NULL) {
611 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
612 /* Local attach case. */
613 if (queue_pair_entry->qp.ref_count > 1) {
614 VMCI_LOG_DEBUG(LGPFX"Error attempting to "
615 "attach more than once.\n");
616 result = VMCI_ERROR_UNAVAILABLE;
617 goto error_keep_entry;
618 }
619
620 if (queue_pair_entry->qp.produce_size != consume_size ||
621 queue_pair_entry->qp.consume_size != produce_size ||
622 queue_pair_entry->qp.flags !=
623 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
624 VMCI_LOG_DEBUG(LGPFX"Error mismatched "
625 "queue pair in local attach.\n");
626 result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
627 goto error_keep_entry;
628 }
629
630 /*
631 * Do a local attach. We swap the consume and produce
632 * queues for the attacher and deliver an attach event.
633 */
634 result = queue_pair_notify_peer_local(true, *handle);
635 if (result < VMCI_SUCCESS)
636 goto error_keep_entry;
637 my_produce_q = queue_pair_entry->consume_q;
638 my_consume_q = queue_pair_entry->produce_q;
639 goto out;
640 }
641 result = VMCI_ERROR_ALREADY_EXISTS;
642 goto error_keep_entry;
643 }
644
645 my_produce_q = vmci_alloc_queue(produce_size, flags);
646 if (!my_produce_q) {
647 VMCI_LOG_WARNING(LGPFX"Error allocating pages for produce "
648 "queue.\n");
649 result = VMCI_ERROR_NO_MEM;
650 goto error;
651 }
652
653 my_consume_q = vmci_alloc_queue(consume_size, flags);
654 if (!my_consume_q) {
655 VMCI_LOG_WARNING(LGPFX"Error allocating pages for consume "
656 "queue.\n");
657 result = VMCI_ERROR_NO_MEM;
658 goto error;
659 }
660
661 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
662 produce_size, consume_size, my_produce_q, my_consume_q);
663 if (!queue_pair_entry) {
664 VMCI_LOG_WARNING(LGPFX"Error allocating memory in %s.\n",
665 __FUNCTION__);
666 result = VMCI_ERROR_NO_MEM;
667 goto error;
668 }
669
670 result = vmci_alloc_ppn_set(my_produce_q, num_produce_pages,
671 my_consume_q, num_consume_pages, &queue_pair_entry->ppn_set);
672 if (result < VMCI_SUCCESS) {
673 VMCI_LOG_WARNING(LGPFX"vmci_alloc_ppn_set failed.\n");
674 goto error;
675 }
676
677 /*
678 * It's only necessary to notify the host if this queue pair will be
679 * attached to from another context.
680 */
681 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
682 /* Local create case. */
683 vmci_id context_id = vmci_get_context_id();
684
685 /*
686 * Enforce similar checks on local queue pairs as we do for
687 * regular ones. The handle's context must match the creator
688 * or attacher context id (here they are both the current
689 * context id) and the attach-only flag cannot exist during
690 * create. We also ensure specified peer is this context or
691 * an invalid one.
692 */
693 if (queue_pair_entry->qp.handle.context != context_id ||
694 (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
695 queue_pair_entry->qp.peer != context_id)) {
696 result = VMCI_ERROR_NO_ACCESS;
697 goto error;
698 }
699
700 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
701 result = VMCI_ERROR_NOT_FOUND;
702 goto error;
703 }
704 } else {
705 result = vmci_queue_pair_alloc_hypercall(queue_pair_entry);
706 if (result < VMCI_SUCCESS) {
707 VMCI_LOG_WARNING(
708 LGPFX"vmci_queue_pair_alloc_hypercall result = "
709 "%d.\n", result);
710 goto error;
711 }
712 }
713
714 queue_pair_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
715
716 out:
717 queue_pair_entry->qp.ref_count++;
718 *handle = queue_pair_entry->qp.handle;
719 *produce_q = (struct vmci_queue *)my_produce_q;
720 *consume_q = (struct vmci_queue *)my_consume_q;
721
722 /*
723 * We should initialize the queue pair header pages on a local queue
724 * pair create. For non-local queue pairs, the hypervisor initializes
725 * the header pages in the create step.
726 */
727 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
728 queue_pair_entry->qp.ref_count == 1) {
729 vmci_queue_header_init((*produce_q)->q_header, *handle);
730 vmci_queue_header_init((*consume_q)->q_header, *handle);
731 }
732
733 vmci_mutex_release(&qp_guest_endpoints.mutex);
734
735 return (VMCI_SUCCESS);
736
737 error:
738 vmci_mutex_release(&qp_guest_endpoints.mutex);
739 if (queue_pair_entry) {
740 /* The queues will be freed inside the destroy routine. */
741 qp_guest_endpoint_destroy(queue_pair_entry);
742 } else {
743 if (my_produce_q)
744 vmci_free_queue(my_produce_q, produce_size);
745 if (my_consume_q)
746 vmci_free_queue(my_consume_q, consume_size);
747 }
748 return (result);
749
750 error_keep_entry:
751 /* This path should only be used when an existing entry was found. */
752 ASSERT(queue_pair_entry->qp.ref_count > 0);
753 vmci_mutex_release(&qp_guest_endpoints.mutex);
754 return (result);
755 }
756
757 /*
758 *------------------------------------------------------------------------------
759 *
760 * vmci_queue_pair_detach_hypercall --
761 *
762 * Helper to make a QueuePairDetach hypercall when the driver is supporting
763 * a guest device.
764 *
765 * Results:
766 * Result of the hypercall.
767 *
768 * Side effects:
769 * None.
770 *
771 *------------------------------------------------------------------------------
772 */
773
774 int
vmci_queue_pair_detach_hypercall(struct vmci_handle handle)775 vmci_queue_pair_detach_hypercall(struct vmci_handle handle)
776 {
777 struct vmci_queue_pair_detach_msg detach_msg;
778
779 detach_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
780 VMCI_QUEUEPAIR_DETACH);
781 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
782 detach_msg.hdr.payload_size = sizeof(handle);
783 detach_msg.handle = handle;
784
785 return (vmci_send_datagram((struct vmci_datagram *)&detach_msg));
786 }
787
788 /*
789 *------------------------------------------------------------------------------
790 *
791 * vmci_queue_pair_detach_guest_work --
792 *
793 * Helper for VMCI QueuePair detach interface. Frees the physical pages for
794 * the queue pair.
795 *
796 * Results:
797 * Success or failure.
798 *
799 * Side effects:
800 * Memory may be freed.
801 *
802 *------------------------------------------------------------------------------
803 */
804
805 static int
vmci_queue_pair_detach_guest_work(struct vmci_handle handle)806 vmci_queue_pair_detach_guest_work(struct vmci_handle handle)
807 {
808 struct qp_guest_endpoint *entry;
809 int result;
810 uint32_t ref_count;
811
812 ASSERT(!VMCI_HANDLE_INVALID(handle));
813
814 vmci_mutex_acquire(&qp_guest_endpoints.mutex);
815
816 entry = (struct qp_guest_endpoint *)queue_pair_list_find_entry(
817 &qp_guest_endpoints, handle);
818 if (!entry) {
819 vmci_mutex_release(&qp_guest_endpoints.mutex);
820 return (VMCI_ERROR_NOT_FOUND);
821 }
822
823 ASSERT(entry->qp.ref_count >= 1);
824
825 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
826 result = VMCI_SUCCESS;
827
828 if (entry->qp.ref_count > 1) {
829 result = queue_pair_notify_peer_local(false, handle);
830
831 /*
832 * We can fail to notify a local queuepair because we
833 * can't allocate. We still want to release the entry
834 * if that happens, so don't bail out yet.
835 */
836 }
837 } else {
838 result = vmci_queue_pair_detach_hypercall(handle);
839 if (entry->hibernate_failure) {
840 if (result == VMCI_ERROR_NOT_FOUND) {
841
842 /*
843 * If a queue pair detach failed when entering
844 * hibernation, the guest driver and the device
845 * may disagree on its existence when coming
846 * out of hibernation. The guest driver will
847 * regard it as a non-local queue pair, but
848 * the device state is gone, since the device
849 * has been powered off. In this case, we
850 * treat the queue pair as a local queue pair
851 * with no peer.
852 */
853
854 ASSERT(entry->qp.ref_count == 1);
855 result = VMCI_SUCCESS;
856 }
857 }
858 if (result < VMCI_SUCCESS) {
859
860 /*
861 * We failed to notify a non-local queuepair. That other
862 * queuepair might still be accessing the shared
863 * memory, so don't release the entry yet. It will get
864 * cleaned up by vmci_queue_pair_Exit() if necessary
865 * (assuming we are going away, otherwise why did this
866 * fail?).
867 */
868
869 vmci_mutex_release(&qp_guest_endpoints.mutex);
870 return (result);
871 }
872 }
873
874 /*
875 * If we get here then we either failed to notify a local queuepair, or
876 * we succeeded in all cases. Release the entry if required.
877 */
878
879 entry->qp.ref_count--;
880 if (entry->qp.ref_count == 0)
881 queue_pair_list_remove_entry(&qp_guest_endpoints, &entry->qp);
882
883 /* If we didn't remove the entry, this could change once we unlock. */
884 ref_count = entry ? entry->qp.ref_count :
885 0xffffffff; /*
886 * Value does not matter, silence the
887 * compiler.
888 */
889
890 vmci_mutex_release(&qp_guest_endpoints.mutex);
891
892 if (ref_count == 0)
893 qp_guest_endpoint_destroy(entry);
894 return (result);
895 }
896
897 /*
898 *------------------------------------------------------------------------------
899 *
900 * queue_pair_notify_peer_local --
901 *
902 * Dispatches a queue pair event message directly into the local event
903 * queue.
904 *
905 * Results:
906 * VMCI_SUCCESS on success, error code otherwise
907 *
908 * Side effects:
909 * None.
910 *
911 *------------------------------------------------------------------------------
912 */
913
914 static int
queue_pair_notify_peer_local(bool attach,struct vmci_handle handle)915 queue_pair_notify_peer_local(bool attach, struct vmci_handle handle)
916 {
917 struct vmci_event_msg *e_msg;
918 struct vmci_event_payload_qp *e_payload;
919 /* buf is only 48 bytes. */
920 vmci_id context_id;
921 context_id = vmci_get_context_id();
922 char buf[sizeof(*e_msg) + sizeof(*e_payload)];
923
924 e_msg = (struct vmci_event_msg *)buf;
925 e_payload = vmci_event_msg_payload(e_msg);
926
927 e_msg->hdr.dst = VMCI_MAKE_HANDLE(context_id, VMCI_EVENT_HANDLER);
928 e_msg->hdr.src = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
929 VMCI_CONTEXT_RESOURCE_ID);
930 e_msg->hdr.payload_size = sizeof(*e_msg) + sizeof(*e_payload) -
931 sizeof(e_msg->hdr);
932 e_msg->event_data.event = attach ? VMCI_EVENT_QP_PEER_ATTACH :
933 VMCI_EVENT_QP_PEER_DETACH;
934 e_payload->peer_id = context_id;
935 e_payload->handle = handle;
936
937 return (vmci_event_dispatch((struct vmci_datagram *)e_msg));
938 }
939