1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011, Bryan Venteicher <[email protected]>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /*
30 * Implements the virtqueue interface as basically described
31 * in the original VirtIO paper.
32 */
33
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/sglist.h>
42 #include <vm/vm.h>
43 #include <vm/pmap.h>
44
45 #include <machine/cpu.h>
46 #include <machine/bus.h>
47 #include <machine/atomic.h>
48 #include <machine/resource.h>
49 #include <sys/bus.h>
50 #include <sys/rman.h>
51
52 #include <dev/virtio/virtio.h>
53 #include <dev/virtio/virtqueue.h>
54 #include <dev/virtio/virtio_ring.h>
55
56 #include "virtio_bus_if.h"
57
58 struct virtqueue {
59 device_t vq_dev;
60 uint16_t vq_queue_index;
61 uint16_t vq_nentries;
62 uint32_t vq_flags;
63 #define VIRTQUEUE_FLAG_MODERN 0x0001
64 #define VIRTQUEUE_FLAG_INDIRECT 0x0002
65 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0004
66
67 int vq_max_indirect_size;
68 bus_size_t vq_notify_offset;
69 virtqueue_intr_t *vq_intrhand;
70 void *vq_intrhand_arg;
71
72 struct vring vq_ring;
73 uint16_t vq_free_cnt;
74 uint16_t vq_queued_cnt;
75 /*
76 * Head of the free chain in the descriptor table. If
77 * there are no free descriptors, this will be set to
78 * VQ_RING_DESC_CHAIN_END.
79 */
80 uint16_t vq_desc_head_idx;
81 /*
82 * Last consumed descriptor in the used table,
83 * trails vq_ring.used->idx.
84 */
85 uint16_t vq_used_cons_idx;
86
87 void *vq_ring_mem;
88 int vq_indirect_mem_size;
89 int vq_alignment;
90 int vq_ring_size;
91 char vq_name[VIRTQUEUE_MAX_NAME_SZ];
92
93 struct vq_desc_extra {
94 void *cookie;
95 struct vring_desc *indirect;
96 vm_paddr_t indirect_paddr;
97 uint16_t ndescs;
98 } vq_descx[0];
99 };
100
101 /*
102 * The maximum virtqueue size is 2^15. Use that value as the end of
103 * descriptor chain terminator since it will never be a valid index
104 * in the descriptor table. This is used to verify we are correctly
105 * handling vq_free_cnt.
106 */
107 #define VQ_RING_DESC_CHAIN_END 32768
108
109 #define VQASSERT(_vq, _exp, _msg, ...) \
110 KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name, \
111 ##__VA_ARGS__))
112
113 #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx) \
114 VQASSERT((_vq), (_idx) < (_vq)->vq_nentries, \
115 "invalid ring index: %d, max: %d", (_idx), \
116 (_vq)->vq_nentries)
117
118 #define VQ_RING_ASSERT_CHAIN_TERM(_vq) \
119 VQASSERT((_vq), (_vq)->vq_desc_head_idx == \
120 VQ_RING_DESC_CHAIN_END, "full ring terminated " \
121 "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
122
123 static int virtqueue_init_indirect(struct virtqueue *vq, int);
124 static void virtqueue_free_indirect(struct virtqueue *vq);
125 static void virtqueue_init_indirect_list(struct virtqueue *,
126 struct vring_desc *);
127
128 static void vq_ring_init(struct virtqueue *);
129 static void vq_ring_update_avail(struct virtqueue *, uint16_t);
130 static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
131 struct vring_desc *, uint16_t, struct sglist *, int, int);
132 static int vq_ring_use_indirect(struct virtqueue *, int);
133 static void vq_ring_enqueue_indirect(struct virtqueue *, void *,
134 struct sglist *, int, int);
135 static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t);
136 static int vq_ring_must_notify_host(struct virtqueue *);
137 static void vq_ring_notify_host(struct virtqueue *);
138 static void vq_ring_free_chain(struct virtqueue *, uint16_t);
139
140 #define vq_modern(_vq) (((_vq)->vq_flags & VIRTQUEUE_FLAG_MODERN) != 0)
141 #define vq_htog16(_vq, _val) virtio_htog16(vq_modern(_vq), _val)
142 #define vq_htog32(_vq, _val) virtio_htog32(vq_modern(_vq), _val)
143 #define vq_htog64(_vq, _val) virtio_htog64(vq_modern(_vq), _val)
144 #define vq_gtoh16(_vq, _val) virtio_gtoh16(vq_modern(_vq), _val)
145 #define vq_gtoh32(_vq, _val) virtio_gtoh32(vq_modern(_vq), _val)
146 #define vq_gtoh64(_vq, _val) virtio_gtoh64(vq_modern(_vq), _val)
147
148 int
virtqueue_alloc(device_t dev,uint16_t queue,uint16_t size,bus_size_t notify_offset,int align,vm_paddr_t highaddr,struct vq_alloc_info * info,struct virtqueue ** vqp)149 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,
150 bus_size_t notify_offset, int align, vm_paddr_t highaddr,
151 struct vq_alloc_info *info, struct virtqueue **vqp)
152 {
153 struct virtqueue *vq;
154 int error;
155
156 *vqp = NULL;
157 error = 0;
158
159 if (size == 0) {
160 device_printf(dev,
161 "virtqueue %d (%s) does not exist (size is zero)\n",
162 queue, info->vqai_name);
163 return (ENODEV);
164 } else if (!powerof2(size)) {
165 device_printf(dev,
166 "virtqueue %d (%s) size is not a power of 2: %d\n",
167 queue, info->vqai_name, size);
168 return (ENXIO);
169 } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
170 device_printf(dev, "virtqueue %d (%s) requested too many "
171 "indirect descriptors: %d, max %d\n",
172 queue, info->vqai_name, info->vqai_maxindirsz,
173 VIRTIO_MAX_INDIRECT);
174 return (EINVAL);
175 }
176
177 vq = malloc(sizeof(struct virtqueue) +
178 size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO);
179 if (vq == NULL) {
180 device_printf(dev, "cannot allocate virtqueue\n");
181 return (ENOMEM);
182 }
183
184 vq->vq_dev = dev;
185 strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
186 vq->vq_queue_index = queue;
187 vq->vq_notify_offset = notify_offset;
188 vq->vq_alignment = align;
189 vq->vq_nentries = size;
190 vq->vq_free_cnt = size;
191 vq->vq_intrhand = info->vqai_intr;
192 vq->vq_intrhand_arg = info->vqai_intr_arg;
193
194 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_F_VERSION_1) != 0)
195 vq->vq_flags |= VIRTQUEUE_FLAG_MODERN;
196 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
197 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
198
199 if (info->vqai_maxindirsz > 1) {
200 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
201 if (error)
202 goto fail;
203 }
204
205 vq->vq_ring_size = round_page(vring_size(size, align));
206 vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
207 M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
208 if (vq->vq_ring_mem == NULL) {
209 device_printf(dev,
210 "cannot allocate memory for virtqueue ring\n");
211 error = ENOMEM;
212 goto fail;
213 }
214
215 vq_ring_init(vq);
216 virtqueue_disable_intr(vq);
217
218 *vqp = vq;
219
220 fail:
221 if (error)
222 virtqueue_free(vq);
223
224 return (error);
225 }
226
227 static int
virtqueue_init_indirect(struct virtqueue * vq,int indirect_size)228 virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
229 {
230 device_t dev;
231 struct vq_desc_extra *dxp;
232 int i, size;
233
234 dev = vq->vq_dev;
235
236 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
237 /*
238 * Indirect descriptors requested by the driver but not
239 * negotiated. Return zero to keep the initialization
240 * going: we'll run fine without.
241 */
242 if (bootverbose)
243 device_printf(dev, "virtqueue %d (%s) requested "
244 "indirect descriptors but not negotiated\n",
245 vq->vq_queue_index, vq->vq_name);
246 return (0);
247 }
248
249 size = indirect_size * sizeof(struct vring_desc);
250 vq->vq_max_indirect_size = indirect_size;
251 vq->vq_indirect_mem_size = size;
252 vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
253
254 for (i = 0; i < vq->vq_nentries; i++) {
255 dxp = &vq->vq_descx[i];
256
257 dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT);
258 if (dxp->indirect == NULL) {
259 device_printf(dev, "cannot allocate indirect list\n");
260 return (ENOMEM);
261 }
262
263 dxp->indirect_paddr = vtophys(dxp->indirect);
264 virtqueue_init_indirect_list(vq, dxp->indirect);
265 }
266
267 return (0);
268 }
269
270 static void
virtqueue_free_indirect(struct virtqueue * vq)271 virtqueue_free_indirect(struct virtqueue *vq)
272 {
273 struct vq_desc_extra *dxp;
274 int i;
275
276 for (i = 0; i < vq->vq_nentries; i++) {
277 dxp = &vq->vq_descx[i];
278
279 if (dxp->indirect == NULL)
280 break;
281
282 free(dxp->indirect, M_DEVBUF);
283 dxp->indirect = NULL;
284 dxp->indirect_paddr = 0;
285 }
286
287 vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
288 vq->vq_indirect_mem_size = 0;
289 }
290
291 static void
virtqueue_init_indirect_list(struct virtqueue * vq,struct vring_desc * indirect)292 virtqueue_init_indirect_list(struct virtqueue *vq,
293 struct vring_desc *indirect)
294 {
295 int i;
296
297 bzero(indirect, vq->vq_indirect_mem_size);
298
299 for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
300 indirect[i].next = vq_gtoh16(vq, i + 1);
301 indirect[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);
302 }
303
304 int
virtqueue_reinit(struct virtqueue * vq,uint16_t size)305 virtqueue_reinit(struct virtqueue *vq, uint16_t size)
306 {
307 struct vq_desc_extra *dxp;
308 int i;
309
310 if (vq->vq_nentries != size) {
311 device_printf(vq->vq_dev,
312 "%s: '%s' changed size; old=%hu, new=%hu\n",
313 __func__, vq->vq_name, vq->vq_nentries, size);
314 return (EINVAL);
315 }
316
317 /* Warn if the virtqueue was not properly cleaned up. */
318 if (vq->vq_free_cnt != vq->vq_nentries) {
319 device_printf(vq->vq_dev,
320 "%s: warning '%s' virtqueue not empty, "
321 "leaking %d entries\n", __func__, vq->vq_name,
322 vq->vq_nentries - vq->vq_free_cnt);
323 }
324
325 vq->vq_desc_head_idx = 0;
326 vq->vq_used_cons_idx = 0;
327 vq->vq_queued_cnt = 0;
328 vq->vq_free_cnt = vq->vq_nentries;
329
330 /* To be safe, reset all our allocated memory. */
331 bzero(vq->vq_ring_mem, vq->vq_ring_size);
332 for (i = 0; i < vq->vq_nentries; i++) {
333 dxp = &vq->vq_descx[i];
334 dxp->cookie = NULL;
335 dxp->ndescs = 0;
336 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
337 virtqueue_init_indirect_list(vq, dxp->indirect);
338 }
339
340 vq_ring_init(vq);
341 virtqueue_disable_intr(vq);
342
343 return (0);
344 }
345
346 void
virtqueue_free(struct virtqueue * vq)347 virtqueue_free(struct virtqueue *vq)
348 {
349
350 if (vq->vq_free_cnt != vq->vq_nentries) {
351 device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
352 "leaking %d entries\n", vq->vq_name,
353 vq->vq_nentries - vq->vq_free_cnt);
354 }
355
356 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
357 virtqueue_free_indirect(vq);
358
359 if (vq->vq_ring_mem != NULL) {
360 contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF);
361 vq->vq_ring_size = 0;
362 vq->vq_ring_mem = NULL;
363 }
364
365 free(vq, M_DEVBUF);
366 }
367
368 vm_paddr_t
virtqueue_paddr(struct virtqueue * vq)369 virtqueue_paddr(struct virtqueue *vq)
370 {
371
372 return (vtophys(vq->vq_ring_mem));
373 }
374
375 vm_paddr_t
virtqueue_desc_paddr(struct virtqueue * vq)376 virtqueue_desc_paddr(struct virtqueue *vq)
377 {
378
379 return (vtophys(vq->vq_ring.desc));
380 }
381
382 vm_paddr_t
virtqueue_avail_paddr(struct virtqueue * vq)383 virtqueue_avail_paddr(struct virtqueue *vq)
384 {
385
386 return (vtophys(vq->vq_ring.avail));
387 }
388
389 vm_paddr_t
virtqueue_used_paddr(struct virtqueue * vq)390 virtqueue_used_paddr(struct virtqueue *vq)
391 {
392
393 return (vtophys(vq->vq_ring.used));
394 }
395
396 uint16_t
virtqueue_index(struct virtqueue * vq)397 virtqueue_index(struct virtqueue *vq)
398 {
399
400 return (vq->vq_queue_index);
401 }
402
403 int
virtqueue_size(struct virtqueue * vq)404 virtqueue_size(struct virtqueue *vq)
405 {
406
407 return (vq->vq_nentries);
408 }
409
410 int
virtqueue_nfree(struct virtqueue * vq)411 virtqueue_nfree(struct virtqueue *vq)
412 {
413
414 return (vq->vq_free_cnt);
415 }
416
417 int
virtqueue_empty(struct virtqueue * vq)418 virtqueue_empty(struct virtqueue *vq)
419 {
420
421 return (vq->vq_nentries == vq->vq_free_cnt);
422 }
423
424 int
virtqueue_full(struct virtqueue * vq)425 virtqueue_full(struct virtqueue *vq)
426 {
427
428 return (vq->vq_free_cnt == 0);
429 }
430
431 void
virtqueue_notify(struct virtqueue * vq)432 virtqueue_notify(struct virtqueue *vq)
433 {
434
435 /* Ensure updated avail->idx is visible to host. */
436 mb();
437
438 if (vq_ring_must_notify_host(vq))
439 vq_ring_notify_host(vq);
440 vq->vq_queued_cnt = 0;
441 }
442
443 int
virtqueue_nused(struct virtqueue * vq)444 virtqueue_nused(struct virtqueue *vq)
445 {
446 uint16_t used_idx, nused;
447
448 used_idx = vq_htog16(vq, vq->vq_ring.used->idx);
449
450 nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
451 VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
452
453 return (nused);
454 }
455
456 int
virtqueue_intr_filter(struct virtqueue * vq)457 virtqueue_intr_filter(struct virtqueue *vq)
458 {
459
460 if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))
461 return (0);
462
463 virtqueue_disable_intr(vq);
464
465 return (1);
466 }
467
468 void
virtqueue_intr(struct virtqueue * vq)469 virtqueue_intr(struct virtqueue *vq)
470 {
471
472 vq->vq_intrhand(vq->vq_intrhand_arg);
473 }
474
475 int
virtqueue_enable_intr(struct virtqueue * vq)476 virtqueue_enable_intr(struct virtqueue *vq)
477 {
478
479 return (vq_ring_enable_interrupt(vq, 0));
480 }
481
482 int
virtqueue_postpone_intr(struct virtqueue * vq,vq_postpone_t hint)483 virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint)
484 {
485 uint16_t ndesc, avail_idx;
486
487 avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
488 ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);
489
490 switch (hint) {
491 case VQ_POSTPONE_SHORT:
492 ndesc = ndesc / 4;
493 break;
494 case VQ_POSTPONE_LONG:
495 ndesc = (ndesc * 3) / 4;
496 break;
497 case VQ_POSTPONE_EMPTIED:
498 break;
499 }
500
501 return (vq_ring_enable_interrupt(vq, ndesc));
502 }
503
504 /*
505 * Note this is only considered a hint to the host.
506 */
507 void
virtqueue_disable_intr(struct virtqueue * vq)508 virtqueue_disable_intr(struct virtqueue *vq)
509 {
510
511 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
512 vring_used_event(&vq->vq_ring) = vq_gtoh16(vq,
513 vq->vq_used_cons_idx - vq->vq_nentries - 1);
514 return;
515 }
516
517 vq->vq_ring.avail->flags |= vq_gtoh16(vq, VRING_AVAIL_F_NO_INTERRUPT);
518 }
519
520 int
virtqueue_enqueue(struct virtqueue * vq,void * cookie,struct sglist * sg,int readable,int writable)521 virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
522 int readable, int writable)
523 {
524 struct vq_desc_extra *dxp;
525 int needed;
526 uint16_t head_idx, idx;
527
528 needed = readable + writable;
529
530 VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
531 VQASSERT(vq, needed == sg->sg_nseg,
532 "segment count mismatch, %d, %d", needed, sg->sg_nseg);
533 VQASSERT(vq,
534 needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
535 "too many segments to enqueue: %d, %d/%d", needed,
536 vq->vq_nentries, vq->vq_max_indirect_size);
537
538 if (needed < 1)
539 return (EINVAL);
540 if (vq->vq_free_cnt == 0)
541 return (ENOSPC);
542
543 if (vq_ring_use_indirect(vq, needed)) {
544 vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
545 return (0);
546 } else if (vq->vq_free_cnt < needed)
547 return (EMSGSIZE);
548
549 head_idx = vq->vq_desc_head_idx;
550 VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
551 dxp = &vq->vq_descx[head_idx];
552
553 VQASSERT(vq, dxp->cookie == NULL,
554 "cookie already exists for index %d", head_idx);
555 dxp->cookie = cookie;
556 dxp->ndescs = needed;
557
558 idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
559 sg, readable, writable);
560
561 vq->vq_desc_head_idx = idx;
562 vq->vq_free_cnt -= needed;
563 if (vq->vq_free_cnt == 0)
564 VQ_RING_ASSERT_CHAIN_TERM(vq);
565 else
566 VQ_RING_ASSERT_VALID_IDX(vq, idx);
567
568 vq_ring_update_avail(vq, head_idx);
569
570 return (0);
571 }
572
573 void *
virtqueue_dequeue(struct virtqueue * vq,uint32_t * len)574 virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
575 {
576 struct vring_used_elem *uep;
577 void *cookie;
578 uint16_t used_idx, desc_idx;
579
580 if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))
581 return (NULL);
582
583 used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
584 uep = &vq->vq_ring.used->ring[used_idx];
585
586 rmb();
587 desc_idx = (uint16_t) vq_htog32(vq, uep->id);
588 if (len != NULL)
589 *len = vq_htog32(vq, uep->len);
590
591 vq_ring_free_chain(vq, desc_idx);
592
593 cookie = vq->vq_descx[desc_idx].cookie;
594 VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
595 vq->vq_descx[desc_idx].cookie = NULL;
596
597 return (cookie);
598 }
599
600 void *
virtqueue_poll(struct virtqueue * vq,uint32_t * len)601 virtqueue_poll(struct virtqueue *vq, uint32_t *len)
602 {
603 void *cookie;
604
605 VIRTIO_BUS_POLL(vq->vq_dev);
606 while ((cookie = virtqueue_dequeue(vq, len)) == NULL) {
607 cpu_spinwait();
608 VIRTIO_BUS_POLL(vq->vq_dev);
609 }
610
611 return (cookie);
612 }
613
614 void *
virtqueue_drain(struct virtqueue * vq,int * last)615 virtqueue_drain(struct virtqueue *vq, int *last)
616 {
617 void *cookie;
618 int idx;
619
620 cookie = NULL;
621 idx = *last;
622
623 while (idx < vq->vq_nentries && cookie == NULL) {
624 if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
625 vq->vq_descx[idx].cookie = NULL;
626 /* Free chain to keep free count consistent. */
627 vq_ring_free_chain(vq, idx);
628 }
629 idx++;
630 }
631
632 *last = idx;
633
634 return (cookie);
635 }
636
637 void
virtqueue_dump(struct virtqueue * vq)638 virtqueue_dump(struct virtqueue *vq)
639 {
640
641 if (vq == NULL)
642 return;
643
644 printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
645 "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
646 "used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
647 vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, virtqueue_nused(vq),
648 vq->vq_queued_cnt, vq->vq_desc_head_idx,
649 vq_htog16(vq, vq->vq_ring.avail->idx), vq->vq_used_cons_idx,
650 vq_htog16(vq, vq->vq_ring.used->idx),
651 vq_htog16(vq, vring_used_event(&vq->vq_ring)),
652 vq_htog16(vq, vq->vq_ring.avail->flags),
653 vq_htog16(vq, vq->vq_ring.used->flags));
654 }
655
656 static void
vq_ring_init(struct virtqueue * vq)657 vq_ring_init(struct virtqueue *vq)
658 {
659 struct vring *vr;
660 char *ring_mem;
661 int i, size;
662
663 ring_mem = vq->vq_ring_mem;
664 size = vq->vq_nentries;
665 vr = &vq->vq_ring;
666
667 vring_init(vr, size, ring_mem, vq->vq_alignment);
668
669 for (i = 0; i < size - 1; i++)
670 vr->desc[i].next = vq_gtoh16(vq, i + 1);
671 vr->desc[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);
672 }
673
674 static void
vq_ring_update_avail(struct virtqueue * vq,uint16_t desc_idx)675 vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
676 {
677 uint16_t avail_idx, avail_ring_idx;
678
679 /*
680 * Place the head of the descriptor chain into the next slot and make
681 * it usable to the host. The chain is made available now rather than
682 * deferring to virtqueue_notify() in the hopes that if the host is
683 * currently running on another CPU, we can keep it processing the new
684 * descriptor.
685 */
686 avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
687 avail_ring_idx = avail_idx & (vq->vq_nentries - 1);
688 vq->vq_ring.avail->ring[avail_ring_idx] = vq_gtoh16(vq, desc_idx);
689
690 wmb();
691 vq->vq_ring.avail->idx = vq_gtoh16(vq, avail_idx + 1);
692
693 /* Keep pending count until virtqueue_notify(). */
694 vq->vq_queued_cnt++;
695 }
696
697 static uint16_t
vq_ring_enqueue_segments(struct virtqueue * vq,struct vring_desc * desc,uint16_t head_idx,struct sglist * sg,int readable,int writable)698 vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
699 uint16_t head_idx, struct sglist *sg, int readable, int writable)
700 {
701 struct sglist_seg *seg;
702 struct vring_desc *dp;
703 int i, needed;
704 uint16_t idx;
705
706 needed = readable + writable;
707
708 for (i = 0, idx = head_idx, seg = sg->sg_segs;
709 i < needed;
710 i++, idx = vq_htog16(vq, dp->next), seg++) {
711 VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
712 "premature end of free desc chain");
713
714 dp = &desc[idx];
715 dp->addr = vq_gtoh64(vq, seg->ss_paddr);
716 dp->len = vq_gtoh32(vq, seg->ss_len);
717 dp->flags = 0;
718
719 if (i < needed - 1)
720 dp->flags |= vq_gtoh16(vq, VRING_DESC_F_NEXT);
721 if (i >= readable)
722 dp->flags |= vq_gtoh16(vq, VRING_DESC_F_WRITE);
723 }
724
725 return (idx);
726 }
727
728 static int
vq_ring_use_indirect(struct virtqueue * vq,int needed)729 vq_ring_use_indirect(struct virtqueue *vq, int needed)
730 {
731
732 if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
733 return (0);
734
735 if (vq->vq_max_indirect_size < needed)
736 return (0);
737
738 if (needed < 2)
739 return (0);
740
741 return (1);
742 }
743
744 static void
vq_ring_enqueue_indirect(struct virtqueue * vq,void * cookie,struct sglist * sg,int readable,int writable)745 vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
746 struct sglist *sg, int readable, int writable)
747 {
748 struct vring_desc *dp;
749 struct vq_desc_extra *dxp;
750 int needed;
751 uint16_t head_idx;
752
753 needed = readable + writable;
754 VQASSERT(vq, needed <= vq->vq_max_indirect_size,
755 "enqueuing too many indirect descriptors");
756
757 head_idx = vq->vq_desc_head_idx;
758 VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
759 dp = &vq->vq_ring.desc[head_idx];
760 dxp = &vq->vq_descx[head_idx];
761
762 VQASSERT(vq, dxp->cookie == NULL,
763 "cookie already exists for index %d", head_idx);
764 dxp->cookie = cookie;
765 dxp->ndescs = 1;
766
767 dp->addr = vq_gtoh64(vq, dxp->indirect_paddr);
768 dp->len = vq_gtoh32(vq, needed * sizeof(struct vring_desc));
769 dp->flags = vq_gtoh16(vq, VRING_DESC_F_INDIRECT);
770
771 vq_ring_enqueue_segments(vq, dxp->indirect, 0,
772 sg, readable, writable);
773
774 vq->vq_desc_head_idx = vq_htog16(vq, dp->next);
775 vq->vq_free_cnt--;
776 if (vq->vq_free_cnt == 0)
777 VQ_RING_ASSERT_CHAIN_TERM(vq);
778 else
779 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
780
781 vq_ring_update_avail(vq, head_idx);
782 }
783
784 static int
vq_ring_enable_interrupt(struct virtqueue * vq,uint16_t ndesc)785 vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc)
786 {
787
788 /*
789 * Enable interrupts, making sure we get the latest index of
790 * what's already been consumed.
791 */
792 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
793 vring_used_event(&vq->vq_ring) =
794 vq_gtoh16(vq, vq->vq_used_cons_idx + ndesc);
795 } else {
796 vq->vq_ring.avail->flags &=
797 vq_gtoh16(vq, ~VRING_AVAIL_F_NO_INTERRUPT);
798 }
799
800 mb();
801
802 /*
803 * Enough items may have already been consumed to meet our threshold
804 * since we last checked. Let our caller know so it processes the new
805 * entries.
806 */
807 if (virtqueue_nused(vq) > ndesc)
808 return (1);
809
810 return (0);
811 }
812
813 static int
vq_ring_must_notify_host(struct virtqueue * vq)814 vq_ring_must_notify_host(struct virtqueue *vq)
815 {
816 uint16_t new_idx, prev_idx, event_idx, flags;
817
818 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
819 new_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
820 prev_idx = new_idx - vq->vq_queued_cnt;
821 event_idx = vq_htog16(vq, vring_avail_event(&vq->vq_ring));
822
823 return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
824 }
825
826 flags = vq->vq_ring.used->flags;
827 return ((flags & vq_gtoh16(vq, VRING_USED_F_NO_NOTIFY)) == 0);
828 }
829
830 static void
vq_ring_notify_host(struct virtqueue * vq)831 vq_ring_notify_host(struct virtqueue *vq)
832 {
833
834 VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index,
835 vq->vq_notify_offset);
836 }
837
838 static void
vq_ring_free_chain(struct virtqueue * vq,uint16_t desc_idx)839 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
840 {
841 struct vring_desc *dp;
842 struct vq_desc_extra *dxp;
843
844 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
845 dp = &vq->vq_ring.desc[desc_idx];
846 dxp = &vq->vq_descx[desc_idx];
847
848 if (vq->vq_free_cnt == 0)
849 VQ_RING_ASSERT_CHAIN_TERM(vq);
850
851 vq->vq_free_cnt += dxp->ndescs;
852 dxp->ndescs--;
853
854 if ((dp->flags & vq_gtoh16(vq, VRING_DESC_F_INDIRECT)) == 0) {
855 while (dp->flags & vq_gtoh16(vq, VRING_DESC_F_NEXT)) {
856 uint16_t next_idx = vq_htog16(vq, dp->next);
857 VQ_RING_ASSERT_VALID_IDX(vq, next_idx);
858 dp = &vq->vq_ring.desc[next_idx];
859 dxp->ndescs--;
860 }
861 }
862
863 VQASSERT(vq, dxp->ndescs == 0,
864 "failed to free entire desc chain, remaining: %d", dxp->ndescs);
865
866 /*
867 * We must append the existing free chain, if any, to the end of
868 * newly freed chain. If the virtqueue was completely used, then
869 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
870 */
871 dp->next = vq_gtoh16(vq, vq->vq_desc_head_idx);
872 vq->vq_desc_head_idx = desc_idx;
873 }
874