1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011, Bryan Venteicher <[email protected]>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /* Driver for VirtIO memory balloon devices. */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/endian.h>
38 #include <sys/kthread.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/sglist.h>
42 #include <sys/sysctl.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/queue.h>
46
47 #include <vm/vm.h>
48 #include <vm/vm_page.h>
49
50 #include <machine/bus.h>
51 #include <machine/resource.h>
52 #include <sys/bus.h>
53 #include <sys/rman.h>
54
55 #include <dev/virtio/virtio.h>
56 #include <dev/virtio/virtqueue.h>
57 #include <dev/virtio/balloon/virtio_balloon.h>
58
59 #include "virtio_if.h"
60
61 struct vtballoon_softc {
62 device_t vtballoon_dev;
63 struct mtx vtballoon_mtx;
64 uint64_t vtballoon_features;
65 uint32_t vtballoon_flags;
66 #define VTBALLOON_FLAG_DETACH 0x01
67
68 struct virtqueue *vtballoon_inflate_vq;
69 struct virtqueue *vtballoon_deflate_vq;
70
71 uint32_t vtballoon_desired_npages;
72 uint32_t vtballoon_current_npages;
73 TAILQ_HEAD(,vm_page) vtballoon_pages;
74
75 struct thread *vtballoon_td;
76 uint32_t *vtballoon_page_frames;
77 int vtballoon_timeout;
78 };
79
80 static struct virtio_feature_desc vtballoon_feature_desc[] = {
81 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" },
82 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" },
83 { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOOM" },
84
85 { 0, NULL }
86 };
87
88 static int vtballoon_probe(device_t);
89 static int vtballoon_attach(device_t);
90 static int vtballoon_detach(device_t);
91 static int vtballoon_config_change(device_t);
92
93 static int vtballoon_negotiate_features(struct vtballoon_softc *);
94 static int vtballoon_setup_features(struct vtballoon_softc *);
95 static int vtballoon_alloc_virtqueues(struct vtballoon_softc *);
96
97 static void vtballoon_vq_intr(void *);
98
99 static void vtballoon_inflate(struct vtballoon_softc *, int);
100 static void vtballoon_deflate(struct vtballoon_softc *, int);
101
102 static void vtballoon_send_page_frames(struct vtballoon_softc *,
103 struct virtqueue *, int);
104
105 static void vtballoon_pop(struct vtballoon_softc *);
106 static void vtballoon_stop(struct vtballoon_softc *);
107
108 static vm_page_t
109 vtballoon_alloc_page(struct vtballoon_softc *);
110 static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
111
112 static int vtballoon_sleep(struct vtballoon_softc *);
113 static void vtballoon_thread(void *);
114 static void vtballoon_setup_sysctl(struct vtballoon_softc *);
115
116 #define vtballoon_modern(_sc) \
117 (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0)
118
119 /* Features desired/implemented by this driver. */
120 #define VTBALLOON_FEATURES VIRTIO_BALLOON_F_MUST_TELL_HOST
121
122 /* Timeout between retries when the balloon needs inflating. */
123 #define VTBALLOON_LOWMEM_TIMEOUT hz
124
125 /*
126 * Maximum number of pages we'll request to inflate or deflate
127 * the balloon in one virtqueue request. Both Linux and NetBSD
128 * have settled on 256, doing up to 1MB at a time.
129 */
130 #define VTBALLOON_PAGES_PER_REQUEST 256
131
132 /* Must be able to fix all pages frames in one page (segment). */
133 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
134
135 #define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx
136 #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \
137 "VirtIO Balloon Lock", MTX_DEF)
138 #define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc)))
139 #define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc)))
140 #define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc)))
141
142 static device_method_t vtballoon_methods[] = {
143 /* Device methods. */
144 DEVMETHOD(device_probe, vtballoon_probe),
145 DEVMETHOD(device_attach, vtballoon_attach),
146 DEVMETHOD(device_detach, vtballoon_detach),
147
148 /* VirtIO methods. */
149 DEVMETHOD(virtio_config_change, vtballoon_config_change),
150
151 DEVMETHOD_END
152 };
153
154 static driver_t vtballoon_driver = {
155 "vtballoon",
156 vtballoon_methods,
157 sizeof(struct vtballoon_softc)
158 };
159 static devclass_t vtballoon_devclass;
160
161 VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver,
162 vtballoon_devclass, 0, 0);
163 MODULE_VERSION(virtio_balloon, 1);
164 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
165
166 VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON,
167 "VirtIO Balloon Adapter");
168
169 static int
vtballoon_probe(device_t dev)170 vtballoon_probe(device_t dev)
171 {
172 return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon));
173 }
174
175 static int
vtballoon_attach(device_t dev)176 vtballoon_attach(device_t dev)
177 {
178 struct vtballoon_softc *sc;
179 int error;
180
181 sc = device_get_softc(dev);
182 sc->vtballoon_dev = dev;
183 virtio_set_feature_desc(dev, vtballoon_feature_desc);
184
185 VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
186 TAILQ_INIT(&sc->vtballoon_pages);
187
188 vtballoon_setup_sysctl(sc);
189
190 error = vtballoon_setup_features(sc);
191 if (error) {
192 device_printf(dev, "cannot setup features\n");
193 goto fail;
194 }
195
196 sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
197 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
198 if (sc->vtballoon_page_frames == NULL) {
199 error = ENOMEM;
200 device_printf(dev,
201 "cannot allocate page frame request array\n");
202 goto fail;
203 }
204
205 error = vtballoon_alloc_virtqueues(sc);
206 if (error) {
207 device_printf(dev, "cannot allocate virtqueues\n");
208 goto fail;
209 }
210
211 error = virtio_setup_intr(dev, INTR_TYPE_MISC);
212 if (error) {
213 device_printf(dev, "cannot setup virtqueue interrupts\n");
214 goto fail;
215 }
216
217 error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
218 0, 0, "virtio_balloon");
219 if (error) {
220 device_printf(dev, "cannot create balloon kthread\n");
221 goto fail;
222 }
223
224 virtqueue_enable_intr(sc->vtballoon_inflate_vq);
225 virtqueue_enable_intr(sc->vtballoon_deflate_vq);
226
227 fail:
228 if (error)
229 vtballoon_detach(dev);
230
231 return (error);
232 }
233
234 static int
vtballoon_detach(device_t dev)235 vtballoon_detach(device_t dev)
236 {
237 struct vtballoon_softc *sc;
238
239 sc = device_get_softc(dev);
240
241 if (sc->vtballoon_td != NULL) {
242 VTBALLOON_LOCK(sc);
243 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
244 wakeup_one(sc);
245 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
246 VTBALLOON_UNLOCK(sc);
247
248 sc->vtballoon_td = NULL;
249 }
250
251 if (device_is_attached(dev)) {
252 vtballoon_pop(sc);
253 vtballoon_stop(sc);
254 }
255
256 if (sc->vtballoon_page_frames != NULL) {
257 free(sc->vtballoon_page_frames, M_DEVBUF);
258 sc->vtballoon_page_frames = NULL;
259 }
260
261 VTBALLOON_LOCK_DESTROY(sc);
262
263 return (0);
264 }
265
266 static int
vtballoon_config_change(device_t dev)267 vtballoon_config_change(device_t dev)
268 {
269 struct vtballoon_softc *sc;
270
271 sc = device_get_softc(dev);
272
273 VTBALLOON_LOCK(sc);
274 wakeup_one(sc);
275 VTBALLOON_UNLOCK(sc);
276
277 return (1);
278 }
279
280 static int
vtballoon_negotiate_features(struct vtballoon_softc * sc)281 vtballoon_negotiate_features(struct vtballoon_softc *sc)
282 {
283 device_t dev;
284 uint64_t features;
285
286 dev = sc->vtballoon_dev;
287 features = VTBALLOON_FEATURES;
288
289 sc->vtballoon_features = virtio_negotiate_features(dev, features);
290 return (virtio_finalize_features(dev));
291 }
292
293 static int
vtballoon_setup_features(struct vtballoon_softc * sc)294 vtballoon_setup_features(struct vtballoon_softc *sc)
295 {
296 int error;
297
298 error = vtballoon_negotiate_features(sc);
299 if (error)
300 return (error);
301
302 return (0);
303 }
304
305 static int
vtballoon_alloc_virtqueues(struct vtballoon_softc * sc)306 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
307 {
308 device_t dev;
309 struct vq_alloc_info vq_info[2];
310 int nvqs;
311
312 dev = sc->vtballoon_dev;
313 nvqs = 2;
314
315 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
316 &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
317
318 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
319 &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
320
321 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
322 }
323
324 static void
vtballoon_vq_intr(void * xsc)325 vtballoon_vq_intr(void *xsc)
326 {
327 struct vtballoon_softc *sc;
328
329 sc = xsc;
330
331 VTBALLOON_LOCK(sc);
332 wakeup_one(sc);
333 VTBALLOON_UNLOCK(sc);
334 }
335
336 static void
vtballoon_inflate(struct vtballoon_softc * sc,int npages)337 vtballoon_inflate(struct vtballoon_softc *sc, int npages)
338 {
339 struct virtqueue *vq;
340 vm_page_t m;
341 int i;
342
343 vq = sc->vtballoon_inflate_vq;
344
345 if (npages > VTBALLOON_PAGES_PER_REQUEST)
346 npages = VTBALLOON_PAGES_PER_REQUEST;
347
348 for (i = 0; i < npages; i++) {
349 if ((m = vtballoon_alloc_page(sc)) == NULL) {
350 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
351 break;
352 }
353
354 sc->vtballoon_page_frames[i] =
355 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
356
357 KASSERT(m->a.queue == PQ_NONE,
358 ("%s: allocated page %p on queue", __func__, m));
359 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
360 }
361
362 if (i > 0)
363 vtballoon_send_page_frames(sc, vq, i);
364 }
365
366 static void
vtballoon_deflate(struct vtballoon_softc * sc,int npages)367 vtballoon_deflate(struct vtballoon_softc *sc, int npages)
368 {
369 TAILQ_HEAD(, vm_page) free_pages;
370 struct virtqueue *vq;
371 vm_page_t m;
372 int i;
373
374 vq = sc->vtballoon_deflate_vq;
375 TAILQ_INIT(&free_pages);
376
377 if (npages > VTBALLOON_PAGES_PER_REQUEST)
378 npages = VTBALLOON_PAGES_PER_REQUEST;
379
380 for (i = 0; i < npages; i++) {
381 m = TAILQ_FIRST(&sc->vtballoon_pages);
382 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
383
384 sc->vtballoon_page_frames[i] =
385 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
386
387 TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
388 TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
389 }
390
391 if (i > 0) {
392 /* Always tell host first before freeing the pages. */
393 vtballoon_send_page_frames(sc, vq, i);
394
395 while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
396 TAILQ_REMOVE(&free_pages, m, plinks.q);
397 vtballoon_free_page(sc, m);
398 }
399 }
400
401 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
402 sc->vtballoon_current_npages == 0) ||
403 (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
404 sc->vtballoon_current_npages != 0),
405 ("%s: bogus page count %d", __func__,
406 sc->vtballoon_current_npages));
407 }
408
409 static void
vtballoon_send_page_frames(struct vtballoon_softc * sc,struct virtqueue * vq,int npages)410 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
411 int npages)
412 {
413 struct sglist sg;
414 struct sglist_seg segs[1];
415 void *c;
416 int error;
417
418 sglist_init(&sg, 1, segs);
419
420 error = sglist_append(&sg, sc->vtballoon_page_frames,
421 npages * sizeof(uint32_t));
422 KASSERT(error == 0, ("error adding page frames to sglist"));
423
424 error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
425 KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
426 virtqueue_notify(vq);
427
428 /*
429 * Inflate and deflate operations are done synchronously. The
430 * interrupt handler will wake us up.
431 */
432 VTBALLOON_LOCK(sc);
433 while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
434 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
435 VTBALLOON_UNLOCK(sc);
436
437 KASSERT(c == vq, ("unexpected balloon operation response"));
438 }
439
440 static void
vtballoon_pop(struct vtballoon_softc * sc)441 vtballoon_pop(struct vtballoon_softc *sc)
442 {
443
444 while (!TAILQ_EMPTY(&sc->vtballoon_pages))
445 vtballoon_deflate(sc, sc->vtballoon_current_npages);
446 }
447
448 static void
vtballoon_stop(struct vtballoon_softc * sc)449 vtballoon_stop(struct vtballoon_softc *sc)
450 {
451
452 virtqueue_disable_intr(sc->vtballoon_inflate_vq);
453 virtqueue_disable_intr(sc->vtballoon_deflate_vq);
454
455 virtio_stop(sc->vtballoon_dev);
456 }
457
458 static vm_page_t
vtballoon_alloc_page(struct vtballoon_softc * sc)459 vtballoon_alloc_page(struct vtballoon_softc *sc)
460 {
461 vm_page_t m;
462
463 m = vm_page_alloc_noobj(VM_ALLOC_NODUMP);
464 if (m != NULL)
465 sc->vtballoon_current_npages++;
466
467 return (m);
468 }
469
470 static void
vtballoon_free_page(struct vtballoon_softc * sc,vm_page_t m)471 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
472 {
473
474 vm_page_free(m);
475 sc->vtballoon_current_npages--;
476 }
477
478 static uint32_t
vtballoon_desired_size(struct vtballoon_softc * sc)479 vtballoon_desired_size(struct vtballoon_softc *sc)
480 {
481 uint32_t desired;
482
483 desired = virtio_read_dev_config_4(sc->vtballoon_dev,
484 offsetof(struct virtio_balloon_config, num_pages));
485
486 if (vtballoon_modern(sc))
487 return (desired);
488 else
489 return (le32toh(desired));
490 }
491
492 static void
vtballoon_update_size(struct vtballoon_softc * sc)493 vtballoon_update_size(struct vtballoon_softc *sc)
494 {
495 uint32_t npages;
496
497 npages = sc->vtballoon_current_npages;
498 if (!vtballoon_modern(sc))
499 npages = htole32(npages);
500
501 virtio_write_dev_config_4(sc->vtballoon_dev,
502 offsetof(struct virtio_balloon_config, actual), npages);
503 }
504
505 static int
vtballoon_sleep(struct vtballoon_softc * sc)506 vtballoon_sleep(struct vtballoon_softc *sc)
507 {
508 int rc, timeout;
509 uint32_t current, desired;
510
511 rc = 0;
512 current = sc->vtballoon_current_npages;
513
514 VTBALLOON_LOCK(sc);
515 for (;;) {
516 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
517 rc = 1;
518 break;
519 }
520
521 desired = vtballoon_desired_size(sc);
522 sc->vtballoon_desired_npages = desired;
523
524 /*
525 * If given, use non-zero timeout on the first time through
526 * the loop. On subsequent times, timeout will be zero so
527 * we will reevaluate the desired size of the balloon and
528 * break out to retry if needed.
529 */
530 timeout = sc->vtballoon_timeout;
531 sc->vtballoon_timeout = 0;
532
533 if (current > desired)
534 break;
535 if (current < desired && timeout == 0)
536 break;
537
538 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
539 }
540 VTBALLOON_UNLOCK(sc);
541
542 return (rc);
543 }
544
545 static void
vtballoon_thread(void * xsc)546 vtballoon_thread(void *xsc)
547 {
548 struct vtballoon_softc *sc;
549 uint32_t current, desired;
550
551 sc = xsc;
552
553 for (;;) {
554 if (vtballoon_sleep(sc) != 0)
555 break;
556
557 current = sc->vtballoon_current_npages;
558 desired = sc->vtballoon_desired_npages;
559
560 if (desired != current) {
561 if (desired > current)
562 vtballoon_inflate(sc, desired - current);
563 else
564 vtballoon_deflate(sc, current - desired);
565
566 vtballoon_update_size(sc);
567 }
568 }
569
570 kthread_exit();
571 }
572
573 static void
vtballoon_setup_sysctl(struct vtballoon_softc * sc)574 vtballoon_setup_sysctl(struct vtballoon_softc *sc)
575 {
576 device_t dev;
577 struct sysctl_ctx_list *ctx;
578 struct sysctl_oid *tree;
579 struct sysctl_oid_list *child;
580
581 dev = sc->vtballoon_dev;
582 ctx = device_get_sysctl_ctx(dev);
583 tree = device_get_sysctl_tree(dev);
584 child = SYSCTL_CHILDREN(tree);
585
586 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
587 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
588 "Desired balloon size in pages");
589
590 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
591 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
592 "Current balloon size in pages");
593 }
594