1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1988, 1991, 1993
5 * The Regents of the University of California. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
32 */
33
34 #include <sys/cdefs.h>
35 #include "opt_param.h"
36 #include "opt_mbuf_stress_test.h"
37 #include "opt_mbuf_profiling.h"
38
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mbuf.h>
46 #include <sys/sysctl.h>
47 #include <sys/domain.h>
48 #include <sys/protosw.h>
49 #include <sys/uio.h>
50 #include <sys/vmmeter.h>
51 #include <sys/sbuf.h>
52 #include <sys/sdt.h>
53 #include <vm/vm.h>
54 #include <vm/vm_pageout.h>
55 #include <vm/vm_page.h>
56
57 SDT_PROBE_DEFINE5_XLATE(sdt, , , m__init,
58 "struct mbuf *", "mbufinfo_t *",
59 "uint32_t", "uint32_t",
60 "uint16_t", "uint16_t",
61 "uint32_t", "uint32_t",
62 "uint32_t", "uint32_t");
63
64 SDT_PROBE_DEFINE3_XLATE(sdt, , , m__gethdr_raw,
65 "uint32_t", "uint32_t",
66 "uint16_t", "uint16_t",
67 "struct mbuf *", "mbufinfo_t *");
68
69 SDT_PROBE_DEFINE3_XLATE(sdt, , , m__gethdr,
70 "uint32_t", "uint32_t",
71 "uint16_t", "uint16_t",
72 "struct mbuf *", "mbufinfo_t *");
73
74 SDT_PROBE_DEFINE3_XLATE(sdt, , , m__get_raw,
75 "uint32_t", "uint32_t",
76 "uint16_t", "uint16_t",
77 "struct mbuf *", "mbufinfo_t *");
78
79 SDT_PROBE_DEFINE3_XLATE(sdt, , , m__get,
80 "uint32_t", "uint32_t",
81 "uint16_t", "uint16_t",
82 "struct mbuf *", "mbufinfo_t *");
83
84 SDT_PROBE_DEFINE4_XLATE(sdt, , , m__getcl,
85 "uint32_t", "uint32_t",
86 "uint16_t", "uint16_t",
87 "uint32_t", "uint32_t",
88 "struct mbuf *", "mbufinfo_t *");
89
90 SDT_PROBE_DEFINE5_XLATE(sdt, , , m__getjcl,
91 "uint32_t", "uint32_t",
92 "uint16_t", "uint16_t",
93 "uint32_t", "uint32_t",
94 "uint32_t", "uint32_t",
95 "struct mbuf *", "mbufinfo_t *");
96
97 SDT_PROBE_DEFINE3_XLATE(sdt, , , m__clget,
98 "struct mbuf *", "mbufinfo_t *",
99 "uint32_t", "uint32_t",
100 "uint32_t", "uint32_t");
101
102 SDT_PROBE_DEFINE4_XLATE(sdt, , , m__cljget,
103 "struct mbuf *", "mbufinfo_t *",
104 "uint32_t", "uint32_t",
105 "uint32_t", "uint32_t",
106 "void*", "void*");
107
108 SDT_PROBE_DEFINE(sdt, , , m__cljset);
109
110 SDT_PROBE_DEFINE1_XLATE(sdt, , , m__free,
111 "struct mbuf *", "mbufinfo_t *");
112
113 SDT_PROBE_DEFINE1_XLATE(sdt, , , m__freem,
114 "struct mbuf *", "mbufinfo_t *");
115
116 SDT_PROBE_DEFINE1_XLATE(sdt, , , m__freemp,
117 "struct mbuf *", "mbufinfo_t *");
118
119 #include <security/mac/mac_framework.h>
120
121 /*
122 * Provide minimum possible defaults for link and protocol header space,
123 * assuming IPv4 over Ethernet. Enabling IPv6, IEEE802.11 or some other
124 * protocol may grow these values.
125 */
126 u_int max_linkhdr = 16;
127 u_int max_protohdr = 40;
128 u_int max_hdr = 16 + 40;
129 SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RD,
130 &max_linkhdr, 16, "Size of largest link layer header");
131 SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RD,
132 &max_protohdr, 40, "Size of largest protocol layer header");
133 SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RD,
134 &max_hdr, 16 + 40, "Size of largest link plus protocol header");
135
136 static void
max_hdr_grow(void)137 max_hdr_grow(void)
138 {
139
140 max_hdr = max_linkhdr + max_protohdr;
141 MPASS(max_hdr <= MHLEN);
142 }
143
144 void
max_linkhdr_grow(u_int new)145 max_linkhdr_grow(u_int new)
146 {
147
148 if (new > max_linkhdr) {
149 max_linkhdr = new;
150 max_hdr_grow();
151 }
152 }
153
154 void
max_protohdr_grow(u_int new)155 max_protohdr_grow(u_int new)
156 {
157
158 if (new > max_protohdr) {
159 max_protohdr = new;
160 max_hdr_grow();
161 }
162 }
163
164 #ifdef MBUF_STRESS_TEST
165 int m_defragpackets;
166 int m_defragbytes;
167 int m_defraguseless;
168 int m_defragfailure;
169 int m_defragrandomfailures;
170
171 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD,
172 &m_defragpackets, 0, "");
173 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD,
174 &m_defragbytes, 0, "");
175 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD,
176 &m_defraguseless, 0, "");
177 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD,
178 &m_defragfailure, 0, "");
179 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
180 &m_defragrandomfailures, 0, "");
181 #endif
182
183 /*
184 * Ensure the correct size of various mbuf parameters. It could be off due
185 * to compiler-induced padding and alignment artifacts.
186 */
187 CTASSERT(MSIZE - offsetof(struct mbuf, m_dat) == MLEN);
188 CTASSERT(MSIZE - offsetof(struct mbuf, m_pktdat) == MHLEN);
189
190 /*
191 * mbuf data storage should be 64-bit aligned regardless of architectural
192 * pointer size; check this is the case with and without a packet header.
193 */
194 CTASSERT(offsetof(struct mbuf, m_dat) % 8 == 0);
195 CTASSERT(offsetof(struct mbuf, m_pktdat) % 8 == 0);
196
197 /*
198 * While the specific values here don't matter too much (i.e., +/- a few
199 * words), we do want to ensure that changes to these values are carefully
200 * reasoned about and properly documented. This is especially the case as
201 * network-protocol and device-driver modules encode these layouts, and must
202 * be recompiled if the structures change. Check these values at compile time
203 * against the ones documented in comments in mbuf.h.
204 *
205 * NB: Possibly they should be documented there via #define's and not just
206 * comments.
207 */
208 #if defined(__LP64__)
209 CTASSERT(offsetof(struct mbuf, m_dat) == 32);
210 CTASSERT(sizeof(struct pkthdr) == 64);
211 CTASSERT(sizeof(struct m_ext) == 160);
212 #else
213 CTASSERT(offsetof(struct mbuf, m_dat) == 24);
214 CTASSERT(sizeof(struct pkthdr) == 56);
215 #if defined(__powerpc__) && defined(BOOKE)
216 /* PowerPC booke has 64-bit physical pointers. */
217 CTASSERT(sizeof(struct m_ext) == 176);
218 #else
219 CTASSERT(sizeof(struct m_ext) == 172);
220 #endif
221 #endif
222
223 /*
224 * Assert that the queue(3) macros produce code of the same size as an old
225 * plain pointer does.
226 */
227 #ifdef INVARIANTS
228 static struct mbuf __used m_assertbuf;
229 CTASSERT(sizeof(m_assertbuf.m_slist) == sizeof(m_assertbuf.m_next));
230 CTASSERT(sizeof(m_assertbuf.m_stailq) == sizeof(m_assertbuf.m_next));
231 CTASSERT(sizeof(m_assertbuf.m_slistpkt) == sizeof(m_assertbuf.m_nextpkt));
232 CTASSERT(sizeof(m_assertbuf.m_stailqpkt) == sizeof(m_assertbuf.m_nextpkt));
233 #endif
234
235 /*
236 * Attach the cluster from *m to *n, set up m_ext in *n
237 * and bump the refcount of the cluster.
238 */
239 void
mb_dupcl(struct mbuf * n,struct mbuf * m)240 mb_dupcl(struct mbuf *n, struct mbuf *m)
241 {
242 volatile u_int *refcnt;
243
244 KASSERT(m->m_flags & (M_EXT | M_EXTPG),
245 ("%s: M_EXT | M_EXTPG not set on %p", __func__, m));
246 KASSERT(!(n->m_flags & (M_EXT | M_EXTPG)),
247 ("%s: M_EXT | M_EXTPG set on %p", __func__, n));
248
249 /*
250 * Cache access optimization.
251 *
252 * o Regular M_EXT storage doesn't need full copy of m_ext, since
253 * the holder of the 'ext_count' is responsible to carry the free
254 * routine and its arguments.
255 * o M_EXTPG data is split between main part of mbuf and m_ext, the
256 * main part is copied in full, the m_ext part is similar to M_EXT.
257 * o EXT_EXTREF, where 'ext_cnt' doesn't point into mbuf at all, is
258 * special - it needs full copy of m_ext into each mbuf, since any
259 * copy could end up as the last to free.
260 */
261 if (m->m_flags & M_EXTPG) {
262 bcopy(&m->m_epg_startcopy, &n->m_epg_startcopy,
263 __rangeof(struct mbuf, m_epg_startcopy, m_epg_endcopy));
264 bcopy(&m->m_ext, &n->m_ext, m_epg_ext_copylen);
265 } else if (m->m_ext.ext_type == EXT_EXTREF)
266 bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext));
267 else
268 bcopy(&m->m_ext, &n->m_ext, m_ext_copylen);
269
270 n->m_flags |= m->m_flags & (M_RDONLY | M_EXT | M_EXTPG);
271
272 /* See if this is the mbuf that holds the embedded refcount. */
273 if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) {
274 refcnt = n->m_ext.ext_cnt = &m->m_ext.ext_count;
275 n->m_ext.ext_flags &= ~EXT_FLAG_EMBREF;
276 } else {
277 KASSERT(m->m_ext.ext_cnt != NULL,
278 ("%s: no refcounting pointer on %p", __func__, m));
279 refcnt = m->m_ext.ext_cnt;
280 }
281
282 if (*refcnt == 1)
283 *refcnt += 1;
284 else
285 atomic_add_int(refcnt, 1);
286 }
287
288 void
m_demote_pkthdr(struct mbuf * m)289 m_demote_pkthdr(struct mbuf *m)
290 {
291
292 M_ASSERTPKTHDR(m);
293 M_ASSERT_NO_SND_TAG(m);
294
295 m_tag_delete_chain(m, NULL);
296 m->m_flags &= ~M_PKTHDR;
297 bzero(&m->m_pkthdr, sizeof(struct pkthdr));
298 }
299
300 /*
301 * Clean up mbuf (chain) from any tags and packet headers.
302 * If "all" is set then the first mbuf in the chain will be
303 * cleaned too.
304 */
305 void
m_demote(struct mbuf * m0,int all,int flags)306 m_demote(struct mbuf *m0, int all, int flags)
307 {
308 struct mbuf *m;
309
310 flags |= M_DEMOTEFLAGS;
311
312 for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) {
313 KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt in m %p, m0 %p",
314 __func__, m, m0));
315 if (m->m_flags & M_PKTHDR)
316 m_demote_pkthdr(m);
317 m->m_flags &= flags;
318 }
319 }
320
321 /*
322 * Sanity checks on mbuf (chain) for use in KASSERT() and general
323 * debugging.
324 * Returns 0 or panics when bad and 1 on all tests passed.
325 * Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they
326 * blow up later.
327 */
328 int
m_sanity(struct mbuf * m0,int sanitize)329 m_sanity(struct mbuf *m0, int sanitize)
330 {
331 struct mbuf *m;
332 caddr_t a, b;
333 int pktlen = 0;
334
335 #ifdef INVARIANTS
336 #define M_SANITY_ACTION(s) panic("mbuf %p: " s, m)
337 #else
338 #define M_SANITY_ACTION(s) printf("mbuf %p: " s, m)
339 #endif
340
341 for (m = m0; m != NULL; m = m->m_next) {
342 /*
343 * Basic pointer checks. If any of these fails then some
344 * unrelated kernel memory before or after us is trashed.
345 * No way to recover from that.
346 */
347 a = M_START(m);
348 b = a + M_SIZE(m);
349 if ((caddr_t)m->m_data < a)
350 M_SANITY_ACTION("m_data outside mbuf data range left");
351 if ((caddr_t)m->m_data > b)
352 M_SANITY_ACTION("m_data outside mbuf data range right");
353 if ((caddr_t)m->m_data + m->m_len > b)
354 M_SANITY_ACTION("m_data + m_len exeeds mbuf space");
355
356 /* m->m_nextpkt may only be set on first mbuf in chain. */
357 if (m != m0 && m->m_nextpkt != NULL) {
358 if (sanitize) {
359 m_freem(m->m_nextpkt);
360 m->m_nextpkt = (struct mbuf *)0xDEADC0DE;
361 } else
362 M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf");
363 }
364
365 /* packet length (not mbuf length!) calculation */
366 if (m0->m_flags & M_PKTHDR)
367 pktlen += m->m_len;
368
369 /* m_tags may only be attached to first mbuf in chain. */
370 if (m != m0 && m->m_flags & M_PKTHDR &&
371 !SLIST_EMPTY(&m->m_pkthdr.tags)) {
372 if (sanitize) {
373 m_tag_delete_chain(m, NULL);
374 /* put in 0xDEADC0DE perhaps? */
375 } else
376 M_SANITY_ACTION("m_tags on in-chain mbuf");
377 }
378
379 /* M_PKTHDR may only be set on first mbuf in chain */
380 if (m != m0 && m->m_flags & M_PKTHDR) {
381 if (sanitize) {
382 bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
383 m->m_flags &= ~M_PKTHDR;
384 /* put in 0xDEADCODE and leave hdr flag in */
385 } else
386 M_SANITY_ACTION("M_PKTHDR on in-chain mbuf");
387 }
388 }
389 m = m0;
390 if (pktlen && pktlen != m->m_pkthdr.len) {
391 if (sanitize)
392 m->m_pkthdr.len = 0;
393 else
394 M_SANITY_ACTION("m_pkthdr.len != mbuf chain length");
395 }
396 return 1;
397
398 #undef M_SANITY_ACTION
399 }
400
401 /*
402 * Non-inlined part of m_init().
403 */
404 int
m_pkthdr_init(struct mbuf * m,int how)405 m_pkthdr_init(struct mbuf *m, int how)
406 {
407 #ifdef MAC
408 int error;
409 #endif
410 m->m_data = m->m_pktdat;
411 bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
412 #ifdef NUMA
413 m->m_pkthdr.numa_domain = M_NODOM;
414 #endif
415 #ifdef MAC
416 /* If the label init fails, fail the alloc */
417 error = mac_mbuf_init(m, how);
418 if (error)
419 return (error);
420 #endif
421
422 return (0);
423 }
424
425 /*
426 * "Move" mbuf pkthdr from "from" to "to".
427 * "from" must have M_PKTHDR set, and "to" must be empty.
428 */
429 void
m_move_pkthdr(struct mbuf * to,struct mbuf * from)430 m_move_pkthdr(struct mbuf *to, struct mbuf *from)
431 {
432
433 #if 0
434 /* see below for why these are not enabled */
435 M_ASSERTPKTHDR(to);
436 /* Note: with MAC, this may not be a good assertion. */
437 KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags),
438 ("m_move_pkthdr: to has tags"));
439 #endif
440 #ifdef MAC
441 /*
442 * XXXMAC: It could be this should also occur for non-MAC?
443 */
444 if (to->m_flags & M_PKTHDR)
445 m_tag_delete_chain(to, NULL);
446 #endif
447 to->m_flags = (from->m_flags & M_COPYFLAGS) |
448 (to->m_flags & (M_EXT | M_EXTPG));
449 if ((to->m_flags & M_EXT) == 0)
450 to->m_data = to->m_pktdat;
451 to->m_pkthdr = from->m_pkthdr; /* especially tags */
452 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
453 from->m_flags &= ~M_PKTHDR;
454 if (from->m_pkthdr.csum_flags & CSUM_SND_TAG) {
455 from->m_pkthdr.csum_flags &= ~CSUM_SND_TAG;
456 from->m_pkthdr.snd_tag = NULL;
457 }
458 }
459
460 /*
461 * Duplicate "from"'s mbuf pkthdr in "to".
462 * "from" must have M_PKTHDR set, and "to" must be empty.
463 * In particular, this does a deep copy of the packet tags.
464 */
465 int
m_dup_pkthdr(struct mbuf * to,const struct mbuf * from,int how)466 m_dup_pkthdr(struct mbuf *to, const struct mbuf *from, int how)
467 {
468
469 #if 0
470 /*
471 * The mbuf allocator only initializes the pkthdr
472 * when the mbuf is allocated with m_gethdr(). Many users
473 * (e.g. m_copy*, m_prepend) use m_get() and then
474 * smash the pkthdr as needed causing these
475 * assertions to trip. For now just disable them.
476 */
477 M_ASSERTPKTHDR(to);
478 /* Note: with MAC, this may not be a good assertion. */
479 KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags"));
480 #endif
481 MBUF_CHECKSLEEP(how);
482 #ifdef MAC
483 if (to->m_flags & M_PKTHDR)
484 m_tag_delete_chain(to, NULL);
485 #endif
486 to->m_flags = (from->m_flags & M_COPYFLAGS) |
487 (to->m_flags & (M_EXT | M_EXTPG));
488 if ((to->m_flags & M_EXT) == 0)
489 to->m_data = to->m_pktdat;
490 to->m_pkthdr = from->m_pkthdr;
491 if (from->m_pkthdr.csum_flags & CSUM_SND_TAG)
492 m_snd_tag_ref(from->m_pkthdr.snd_tag);
493 SLIST_INIT(&to->m_pkthdr.tags);
494 return (m_tag_copy_chain(to, from, how));
495 }
496
497 /*
498 * Lesser-used path for M_PREPEND:
499 * allocate new mbuf to prepend to chain,
500 * copy junk along.
501 */
502 struct mbuf *
m_prepend(struct mbuf * m,int len,int how)503 m_prepend(struct mbuf *m, int len, int how)
504 {
505 struct mbuf *mn;
506
507 if (m->m_flags & M_PKTHDR)
508 mn = m_gethdr(how, m->m_type);
509 else
510 mn = m_get(how, m->m_type);
511 if (mn == NULL) {
512 m_freem(m);
513 return (NULL);
514 }
515 if (m->m_flags & M_PKTHDR)
516 m_move_pkthdr(mn, m);
517 mn->m_next = m;
518 m = mn;
519 if (len < M_SIZE(m))
520 M_ALIGN(m, len);
521 m->m_len = len;
522 return (m);
523 }
524
525 /*
526 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
527 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
528 * The wait parameter is a choice of M_WAITOK/M_NOWAIT from caller.
529 * Note that the copy is read-only, because clusters are not copied,
530 * only their reference counts are incremented.
531 */
532 struct mbuf *
m_copym(struct mbuf * m,int off0,int len,int wait)533 m_copym(struct mbuf *m, int off0, int len, int wait)
534 {
535 struct mbuf *n, **np;
536 int off = off0;
537 struct mbuf *top;
538 int copyhdr = 0;
539
540 KASSERT(off >= 0, ("m_copym, negative off %d", off));
541 KASSERT(len >= 0, ("m_copym, negative len %d", len));
542 MBUF_CHECKSLEEP(wait);
543 if (off == 0 && m->m_flags & M_PKTHDR)
544 copyhdr = 1;
545 while (off > 0) {
546 KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
547 if (off < m->m_len)
548 break;
549 off -= m->m_len;
550 m = m->m_next;
551 }
552 np = ⊤
553 top = NULL;
554 while (len > 0) {
555 if (m == NULL) {
556 KASSERT(len == M_COPYALL,
557 ("m_copym, length > size of mbuf chain"));
558 break;
559 }
560 if (copyhdr)
561 n = m_gethdr(wait, m->m_type);
562 else
563 n = m_get(wait, m->m_type);
564 *np = n;
565 if (n == NULL)
566 goto nospace;
567 if (copyhdr) {
568 if (!m_dup_pkthdr(n, m, wait))
569 goto nospace;
570 if (len == M_COPYALL)
571 n->m_pkthdr.len -= off0;
572 else
573 n->m_pkthdr.len = len;
574 copyhdr = 0;
575 }
576 n->m_len = min(len, m->m_len - off);
577 if (m->m_flags & (M_EXT | M_EXTPG)) {
578 n->m_data = m->m_data + off;
579 mb_dupcl(n, m);
580 } else
581 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
582 (u_int)n->m_len);
583 if (len != M_COPYALL)
584 len -= n->m_len;
585 off = 0;
586 m = m->m_next;
587 np = &n->m_next;
588 }
589
590 return (top);
591 nospace:
592 m_freem(top);
593 return (NULL);
594 }
595
596 /*
597 * Copy an entire packet, including header (which must be present).
598 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
599 * Note that the copy is read-only, because clusters are not copied,
600 * only their reference counts are incremented.
601 * Preserve alignment of the first mbuf so if the creator has left
602 * some room at the beginning (e.g. for inserting protocol headers)
603 * the copies still have the room available.
604 */
605 struct mbuf *
m_copypacket(struct mbuf * m,int how)606 m_copypacket(struct mbuf *m, int how)
607 {
608 struct mbuf *top, *n, *o;
609
610 MBUF_CHECKSLEEP(how);
611 n = m_get(how, m->m_type);
612 top = n;
613 if (n == NULL)
614 goto nospace;
615
616 if (!m_dup_pkthdr(n, m, how))
617 goto nospace;
618 n->m_len = m->m_len;
619 if (m->m_flags & (M_EXT | M_EXTPG)) {
620 n->m_data = m->m_data;
621 mb_dupcl(n, m);
622 } else {
623 n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
624 bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
625 }
626
627 m = m->m_next;
628 while (m) {
629 o = m_get(how, m->m_type);
630 if (o == NULL)
631 goto nospace;
632
633 n->m_next = o;
634 n = n->m_next;
635
636 n->m_len = m->m_len;
637 if (m->m_flags & (M_EXT | M_EXTPG)) {
638 n->m_data = m->m_data;
639 mb_dupcl(n, m);
640 } else {
641 bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
642 }
643
644 m = m->m_next;
645 }
646 return top;
647 nospace:
648 m_freem(top);
649 return (NULL);
650 }
651
652 static void
m_copyfromunmapped(const struct mbuf * m,int off,int len,caddr_t cp)653 m_copyfromunmapped(const struct mbuf *m, int off, int len, caddr_t cp)
654 {
655 struct iovec iov;
656 struct uio uio;
657 int error __diagused;
658
659 KASSERT(off >= 0, ("m_copyfromunmapped: negative off %d", off));
660 KASSERT(len >= 0, ("m_copyfromunmapped: negative len %d", len));
661 KASSERT(off < m->m_len,
662 ("m_copyfromunmapped: len exceeds mbuf length"));
663 iov.iov_base = cp;
664 iov.iov_len = len;
665 uio.uio_resid = len;
666 uio.uio_iov = &iov;
667 uio.uio_segflg = UIO_SYSSPACE;
668 uio.uio_iovcnt = 1;
669 uio.uio_offset = 0;
670 uio.uio_rw = UIO_READ;
671 error = m_unmapped_uiomove(m, off, &uio, len);
672 KASSERT(error == 0, ("m_unmapped_uiomove failed: off %d, len %d", off,
673 len));
674 }
675
676 /*
677 * Copy data from an mbuf chain starting "off" bytes from the beginning,
678 * continuing for "len" bytes, into the indicated buffer.
679 */
680 void
m_copydata(const struct mbuf * m,int off,int len,caddr_t cp)681 m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
682 {
683 u_int count;
684
685 KASSERT(off >= 0, ("m_copydata, negative off %d", off));
686 KASSERT(len >= 0, ("m_copydata, negative len %d", len));
687 while (off > 0) {
688 KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
689 if (off < m->m_len)
690 break;
691 off -= m->m_len;
692 m = m->m_next;
693 }
694 while (len > 0) {
695 KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
696 count = min(m->m_len - off, len);
697 if ((m->m_flags & M_EXTPG) != 0)
698 m_copyfromunmapped(m, off, count, cp);
699 else
700 bcopy(mtod(m, caddr_t) + off, cp, count);
701 len -= count;
702 cp += count;
703 off = 0;
704 m = m->m_next;
705 }
706 }
707
708 /*
709 * Copy a packet header mbuf chain into a completely new chain, including
710 * copying any mbuf clusters. Use this instead of m_copypacket() when
711 * you need a writable copy of an mbuf chain.
712 */
713 struct mbuf *
m_dup(const struct mbuf * m,int how)714 m_dup(const struct mbuf *m, int how)
715 {
716 struct mbuf **p, *top = NULL;
717 int remain, moff, nsize;
718
719 MBUF_CHECKSLEEP(how);
720 /* Sanity check */
721 if (m == NULL)
722 return (NULL);
723 M_ASSERTPKTHDR(m);
724
725 /* While there's more data, get a new mbuf, tack it on, and fill it */
726 remain = m->m_pkthdr.len;
727 moff = 0;
728 p = ⊤
729 while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */
730 struct mbuf *n;
731
732 /* Get the next new mbuf */
733 if (remain >= MINCLSIZE) {
734 n = m_getcl(how, m->m_type, 0);
735 nsize = MCLBYTES;
736 } else {
737 n = m_get(how, m->m_type);
738 nsize = MLEN;
739 }
740 if (n == NULL)
741 goto nospace;
742
743 if (top == NULL) { /* First one, must be PKTHDR */
744 if (!m_dup_pkthdr(n, m, how)) {
745 m_free(n);
746 goto nospace;
747 }
748 if ((n->m_flags & M_EXT) == 0)
749 nsize = MHLEN;
750 n->m_flags &= ~M_RDONLY;
751 }
752 n->m_len = 0;
753
754 /* Link it into the new chain */
755 *p = n;
756 p = &n->m_next;
757
758 /* Copy data from original mbuf(s) into new mbuf */
759 while (n->m_len < nsize && m != NULL) {
760 int chunk = min(nsize - n->m_len, m->m_len - moff);
761
762 m_copydata(m, moff, chunk, n->m_data + n->m_len);
763 moff += chunk;
764 n->m_len += chunk;
765 remain -= chunk;
766 if (moff == m->m_len) {
767 m = m->m_next;
768 moff = 0;
769 }
770 }
771
772 /* Check correct total mbuf length */
773 KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL),
774 ("%s: bogus m_pkthdr.len", __func__));
775 }
776 return (top);
777
778 nospace:
779 m_freem(top);
780 return (NULL);
781 }
782
783 /*
784 * Concatenate mbuf chain n to m.
785 * Both chains must be of the same type (e.g. MT_DATA).
786 * Any m_pkthdr is not updated.
787 */
788 void
m_cat(struct mbuf * m,struct mbuf * n)789 m_cat(struct mbuf *m, struct mbuf *n)
790 {
791 while (m->m_next)
792 m = m->m_next;
793 while (n) {
794 if (!M_WRITABLE(m) ||
795 (n->m_flags & M_EXTPG) != 0 ||
796 M_TRAILINGSPACE(m) < n->m_len) {
797 /* just join the two chains */
798 m->m_next = n;
799 return;
800 }
801 /* splat the data from one into the other */
802 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
803 (u_int)n->m_len);
804 m->m_len += n->m_len;
805 n = m_free(n);
806 }
807 }
808
809 /*
810 * Concatenate two pkthdr mbuf chains.
811 */
812 void
m_catpkt(struct mbuf * m,struct mbuf * n)813 m_catpkt(struct mbuf *m, struct mbuf *n)
814 {
815
816 M_ASSERTPKTHDR(m);
817 M_ASSERTPKTHDR(n);
818
819 m->m_pkthdr.len += n->m_pkthdr.len;
820 m_demote(n, 1, 0);
821
822 m_cat(m, n);
823 }
824
825 void
m_adj(struct mbuf * mp,int req_len)826 m_adj(struct mbuf *mp, int req_len)
827 {
828 int len = req_len;
829 struct mbuf *m;
830 int count;
831
832 if ((m = mp) == NULL)
833 return;
834 if (len >= 0) {
835 /*
836 * Trim from head.
837 */
838 while (m != NULL && len > 0) {
839 if (m->m_len <= len) {
840 len -= m->m_len;
841 m->m_len = 0;
842 m = m->m_next;
843 } else {
844 m->m_len -= len;
845 m->m_data += len;
846 len = 0;
847 }
848 }
849 if (mp->m_flags & M_PKTHDR)
850 mp->m_pkthdr.len -= (req_len - len);
851 } else {
852 /*
853 * Trim from tail. Scan the mbuf chain,
854 * calculating its length and finding the last mbuf.
855 * If the adjustment only affects this mbuf, then just
856 * adjust and return. Otherwise, rescan and truncate
857 * after the remaining size.
858 */
859 len = -len;
860 count = 0;
861 for (;;) {
862 count += m->m_len;
863 if (m->m_next == (struct mbuf *)0)
864 break;
865 m = m->m_next;
866 }
867 if (m->m_len >= len) {
868 m->m_len -= len;
869 if (mp->m_flags & M_PKTHDR)
870 mp->m_pkthdr.len -= len;
871 return;
872 }
873 count -= len;
874 if (count < 0)
875 count = 0;
876 /*
877 * Correct length for chain is "count".
878 * Find the mbuf with last data, adjust its length,
879 * and toss data from remaining mbufs on chain.
880 */
881 m = mp;
882 if (m->m_flags & M_PKTHDR)
883 m->m_pkthdr.len = count;
884 for (; m; m = m->m_next) {
885 if (m->m_len >= count) {
886 m->m_len = count;
887 if (m->m_next != NULL) {
888 m_freem(m->m_next);
889 m->m_next = NULL;
890 }
891 break;
892 }
893 count -= m->m_len;
894 }
895 }
896 }
897
898 void
m_adj_decap(struct mbuf * mp,int len)899 m_adj_decap(struct mbuf *mp, int len)
900 {
901 uint8_t rsstype;
902
903 m_adj(mp, len);
904 if ((mp->m_flags & M_PKTHDR) != 0) {
905 /*
906 * If flowid was calculated by card from the inner
907 * headers, move flowid to the decapsulated mbuf
908 * chain, otherwise clear. This depends on the
909 * internals of m_adj, which keeps pkthdr as is, in
910 * particular not changing rsstype and flowid.
911 */
912 rsstype = mp->m_pkthdr.rsstype;
913 if ((rsstype & M_HASHTYPE_INNER) != 0) {
914 M_HASHTYPE_SET(mp, rsstype & ~M_HASHTYPE_INNER);
915 } else {
916 M_HASHTYPE_CLEAR(mp);
917 }
918 }
919 }
920
921 /*
922 * Rearange an mbuf chain so that len bytes are contiguous
923 * and in the data area of an mbuf (so that mtod will work
924 * for a structure of size len). Returns the resulting
925 * mbuf chain on success, frees it and returns null on failure.
926 * If there is room, it will add up to max_protohdr-len extra bytes to the
927 * contiguous region in an attempt to avoid being called next time.
928 */
929 struct mbuf *
m_pullup(struct mbuf * n,int len)930 m_pullup(struct mbuf *n, int len)
931 {
932 struct mbuf *m;
933 int count;
934 int space;
935
936 KASSERT((n->m_flags & M_EXTPG) == 0,
937 ("%s: unmapped mbuf %p", __func__, n));
938
939 /*
940 * If first mbuf has no cluster, and has room for len bytes
941 * without shifting current data, pullup into it,
942 * otherwise allocate a new mbuf to prepend to the chain.
943 */
944 if ((n->m_flags & M_EXT) == 0 &&
945 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
946 if (n->m_len >= len)
947 return (n);
948 m = n;
949 n = n->m_next;
950 len -= m->m_len;
951 } else {
952 if (len > MHLEN)
953 goto bad;
954 m = m_get(M_NOWAIT, n->m_type);
955 if (m == NULL)
956 goto bad;
957 if (n->m_flags & M_PKTHDR)
958 m_move_pkthdr(m, n);
959 }
960 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
961 do {
962 count = min(min(max(len, max_protohdr), space), n->m_len);
963 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
964 (u_int)count);
965 len -= count;
966 m->m_len += count;
967 n->m_len -= count;
968 space -= count;
969 if (n->m_len)
970 n->m_data += count;
971 else
972 n = m_free(n);
973 } while (len > 0 && n);
974 if (len > 0) {
975 (void) m_free(m);
976 goto bad;
977 }
978 m->m_next = n;
979 return (m);
980 bad:
981 m_freem(n);
982 return (NULL);
983 }
984
985 /*
986 * Like m_pullup(), except a new mbuf is always allocated, and we allow
987 * the amount of empty space before the data in the new mbuf to be specified
988 * (in the event that the caller expects to prepend later).
989 */
990 struct mbuf *
m_copyup(struct mbuf * n,int len,int dstoff)991 m_copyup(struct mbuf *n, int len, int dstoff)
992 {
993 struct mbuf *m;
994 int count, space;
995
996 if (len > (MHLEN - dstoff))
997 goto bad;
998 m = m_get(M_NOWAIT, n->m_type);
999 if (m == NULL)
1000 goto bad;
1001 if (n->m_flags & M_PKTHDR)
1002 m_move_pkthdr(m, n);
1003 m->m_data += dstoff;
1004 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1005 do {
1006 count = min(min(max(len, max_protohdr), space), n->m_len);
1007 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
1008 (unsigned)count);
1009 len -= count;
1010 m->m_len += count;
1011 n->m_len -= count;
1012 space -= count;
1013 if (n->m_len)
1014 n->m_data += count;
1015 else
1016 n = m_free(n);
1017 } while (len > 0 && n);
1018 if (len > 0) {
1019 (void) m_free(m);
1020 goto bad;
1021 }
1022 m->m_next = n;
1023 return (m);
1024 bad:
1025 m_freem(n);
1026 return (NULL);
1027 }
1028
1029 /*
1030 * Partition an mbuf chain in two pieces, returning the tail --
1031 * all but the first len0 bytes. In case of failure, it returns NULL and
1032 * attempts to restore the chain to its original state.
1033 *
1034 * Note that the resulting mbufs might be read-only, because the new
1035 * mbuf can end up sharing an mbuf cluster with the original mbuf if
1036 * the "breaking point" happens to lie within a cluster mbuf. Use the
1037 * M_WRITABLE() macro to check for this case.
1038 */
1039 struct mbuf *
m_split(struct mbuf * m0,int len0,int wait)1040 m_split(struct mbuf *m0, int len0, int wait)
1041 {
1042 struct mbuf *m, *n;
1043 u_int len = len0, remain;
1044
1045 MBUF_CHECKSLEEP(wait);
1046 for (m = m0; m && len > m->m_len; m = m->m_next)
1047 len -= m->m_len;
1048 if (m == NULL)
1049 return (NULL);
1050 remain = m->m_len - len;
1051 if (m0->m_flags & M_PKTHDR && remain == 0) {
1052 n = m_gethdr(wait, m0->m_type);
1053 if (n == NULL)
1054 return (NULL);
1055 n->m_next = m->m_next;
1056 m->m_next = NULL;
1057 if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) {
1058 n->m_pkthdr.snd_tag =
1059 m_snd_tag_ref(m0->m_pkthdr.snd_tag);
1060 n->m_pkthdr.csum_flags |= CSUM_SND_TAG;
1061 } else
1062 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1063 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1064 m0->m_pkthdr.len = len0;
1065 return (n);
1066 } else if (m0->m_flags & M_PKTHDR) {
1067 n = m_gethdr(wait, m0->m_type);
1068 if (n == NULL)
1069 return (NULL);
1070 if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) {
1071 n->m_pkthdr.snd_tag =
1072 m_snd_tag_ref(m0->m_pkthdr.snd_tag);
1073 n->m_pkthdr.csum_flags |= CSUM_SND_TAG;
1074 } else
1075 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1076 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1077 m0->m_pkthdr.len = len0;
1078 if (m->m_flags & (M_EXT | M_EXTPG))
1079 goto extpacket;
1080 if (remain > MHLEN) {
1081 /* m can't be the lead packet */
1082 M_ALIGN(n, 0);
1083 n->m_next = m_split(m, len, wait);
1084 if (n->m_next == NULL) {
1085 (void) m_free(n);
1086 return (NULL);
1087 } else {
1088 n->m_len = 0;
1089 return (n);
1090 }
1091 } else
1092 M_ALIGN(n, remain);
1093 } else if (remain == 0) {
1094 n = m->m_next;
1095 m->m_next = NULL;
1096 return (n);
1097 } else {
1098 n = m_get(wait, m->m_type);
1099 if (n == NULL)
1100 return (NULL);
1101 M_ALIGN(n, remain);
1102 }
1103 extpacket:
1104 if (m->m_flags & (M_EXT | M_EXTPG)) {
1105 n->m_data = m->m_data + len;
1106 mb_dupcl(n, m);
1107 } else {
1108 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1109 }
1110 n->m_len = remain;
1111 m->m_len = len;
1112 n->m_next = m->m_next;
1113 m->m_next = NULL;
1114 return (n);
1115 }
1116 /*
1117 * Routine to copy from device local memory into mbufs.
1118 * Note that `off' argument is offset into first mbuf of target chain from
1119 * which to begin copying the data to.
1120 */
1121 struct mbuf *
m_devget(char * buf,int totlen,int off,struct ifnet * ifp,void (* copy)(char * from,caddr_t to,u_int len))1122 m_devget(char *buf, int totlen, int off, struct ifnet *ifp,
1123 void (*copy)(char *from, caddr_t to, u_int len))
1124 {
1125 struct mbuf *m;
1126 struct mbuf *top = NULL, **mp = ⊤
1127 int len;
1128
1129 if (off < 0 || off > MHLEN)
1130 return (NULL);
1131
1132 while (totlen > 0) {
1133 if (top == NULL) { /* First one, must be PKTHDR */
1134 if (totlen + off >= MINCLSIZE) {
1135 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1136 len = MCLBYTES;
1137 } else {
1138 m = m_gethdr(M_NOWAIT, MT_DATA);
1139 len = MHLEN;
1140
1141 /* Place initial small packet/header at end of mbuf */
1142 if (m && totlen + off + max_linkhdr <= MHLEN) {
1143 m->m_data += max_linkhdr;
1144 len -= max_linkhdr;
1145 }
1146 }
1147 if (m == NULL)
1148 return NULL;
1149 m->m_pkthdr.rcvif = ifp;
1150 m->m_pkthdr.len = totlen;
1151 } else {
1152 if (totlen + off >= MINCLSIZE) {
1153 m = m_getcl(M_NOWAIT, MT_DATA, 0);
1154 len = MCLBYTES;
1155 } else {
1156 m = m_get(M_NOWAIT, MT_DATA);
1157 len = MLEN;
1158 }
1159 if (m == NULL) {
1160 m_freem(top);
1161 return NULL;
1162 }
1163 }
1164 if (off) {
1165 m->m_data += off;
1166 len -= off;
1167 off = 0;
1168 }
1169 m->m_len = len = min(totlen, len);
1170 if (copy)
1171 copy(buf, mtod(m, caddr_t), (u_int)len);
1172 else
1173 bcopy(buf, mtod(m, caddr_t), (u_int)len);
1174 buf += len;
1175 *mp = m;
1176 mp = &m->m_next;
1177 totlen -= len;
1178 }
1179 return (top);
1180 }
1181
1182 static void
m_copytounmapped(const struct mbuf * m,int off,int len,c_caddr_t cp)1183 m_copytounmapped(const struct mbuf *m, int off, int len, c_caddr_t cp)
1184 {
1185 struct iovec iov;
1186 struct uio uio;
1187 int error __diagused;
1188
1189 KASSERT(off >= 0, ("m_copytounmapped: negative off %d", off));
1190 KASSERT(len >= 0, ("m_copytounmapped: negative len %d", len));
1191 KASSERT(off < m->m_len, ("m_copytounmapped: len exceeds mbuf length"));
1192 iov.iov_base = __DECONST(caddr_t, cp);
1193 iov.iov_len = len;
1194 uio.uio_resid = len;
1195 uio.uio_iov = &iov;
1196 uio.uio_segflg = UIO_SYSSPACE;
1197 uio.uio_iovcnt = 1;
1198 uio.uio_offset = 0;
1199 uio.uio_rw = UIO_WRITE;
1200 error = m_unmapped_uiomove(m, off, &uio, len);
1201 KASSERT(error == 0, ("m_unmapped_uiomove failed: off %d, len %d", off,
1202 len));
1203 }
1204
1205 /*
1206 * Copy data from a buffer back into the indicated mbuf chain,
1207 * starting "off" bytes from the beginning, extending the mbuf
1208 * chain if necessary.
1209 */
1210 void
m_copyback(struct mbuf * m0,int off,int len,c_caddr_t cp)1211 m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp)
1212 {
1213 int mlen;
1214 struct mbuf *m = m0, *n;
1215 int totlen = 0;
1216
1217 if (m0 == NULL)
1218 return;
1219 while (off > (mlen = m->m_len)) {
1220 off -= mlen;
1221 totlen += mlen;
1222 if (m->m_next == NULL) {
1223 n = m_get(M_NOWAIT, m->m_type);
1224 if (n == NULL)
1225 goto out;
1226 bzero(mtod(n, caddr_t), MLEN);
1227 n->m_len = min(MLEN, len + off);
1228 m->m_next = n;
1229 }
1230 m = m->m_next;
1231 }
1232 while (len > 0) {
1233 if (m->m_next == NULL && (len > m->m_len - off)) {
1234 m->m_len += min(len - (m->m_len - off),
1235 M_TRAILINGSPACE(m));
1236 }
1237 mlen = min (m->m_len - off, len);
1238 if ((m->m_flags & M_EXTPG) != 0)
1239 m_copytounmapped(m, off, mlen, cp);
1240 else
1241 bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen);
1242 cp += mlen;
1243 len -= mlen;
1244 mlen += off;
1245 off = 0;
1246 totlen += mlen;
1247 if (len == 0)
1248 break;
1249 if (m->m_next == NULL) {
1250 n = m_get(M_NOWAIT, m->m_type);
1251 if (n == NULL)
1252 break;
1253 n->m_len = min(MLEN, len);
1254 m->m_next = n;
1255 }
1256 m = m->m_next;
1257 }
1258 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1259 m->m_pkthdr.len = totlen;
1260 }
1261
1262 /*
1263 * Append the specified data to the indicated mbuf chain,
1264 * Extend the mbuf chain if the new data does not fit in
1265 * existing space.
1266 *
1267 * Return 1 if able to complete the job; otherwise 0.
1268 */
1269 int
m_append(struct mbuf * m0,int len,c_caddr_t cp)1270 m_append(struct mbuf *m0, int len, c_caddr_t cp)
1271 {
1272 struct mbuf *m, *n;
1273 int remainder, space;
1274
1275 for (m = m0; m->m_next != NULL; m = m->m_next)
1276 ;
1277 remainder = len;
1278 space = M_TRAILINGSPACE(m);
1279 if (space > 0) {
1280 /*
1281 * Copy into available space.
1282 */
1283 if (space > remainder)
1284 space = remainder;
1285 bcopy(cp, mtod(m, caddr_t) + m->m_len, space);
1286 m->m_len += space;
1287 cp += space, remainder -= space;
1288 }
1289 while (remainder > 0) {
1290 /*
1291 * Allocate a new mbuf; could check space
1292 * and allocate a cluster instead.
1293 */
1294 n = m_get(M_NOWAIT, m->m_type);
1295 if (n == NULL)
1296 break;
1297 n->m_len = min(MLEN, remainder);
1298 bcopy(cp, mtod(n, caddr_t), n->m_len);
1299 cp += n->m_len, remainder -= n->m_len;
1300 m->m_next = n;
1301 m = n;
1302 }
1303 if (m0->m_flags & M_PKTHDR)
1304 m0->m_pkthdr.len += len - remainder;
1305 return (remainder == 0);
1306 }
1307
1308 static int
m_apply_extpg_one(struct mbuf * m,int off,int len,int (* f)(void *,void *,u_int),void * arg)1309 m_apply_extpg_one(struct mbuf *m, int off, int len,
1310 int (*f)(void *, void *, u_int), void *arg)
1311 {
1312 void *p;
1313 u_int i, count, pgoff, pglen;
1314 int rval;
1315
1316 KASSERT(PMAP_HAS_DMAP,
1317 ("m_apply_extpg_one does not support unmapped mbufs"));
1318 off += mtod(m, vm_offset_t);
1319 if (off < m->m_epg_hdrlen) {
1320 count = min(m->m_epg_hdrlen - off, len);
1321 rval = f(arg, m->m_epg_hdr + off, count);
1322 if (rval)
1323 return (rval);
1324 len -= count;
1325 off = 0;
1326 } else
1327 off -= m->m_epg_hdrlen;
1328 pgoff = m->m_epg_1st_off;
1329 for (i = 0; i < m->m_epg_npgs && len > 0; i++) {
1330 pglen = m_epg_pagelen(m, i, pgoff);
1331 if (off < pglen) {
1332 count = min(pglen - off, len);
1333 p = (void *)PHYS_TO_DMAP(m->m_epg_pa[i] + pgoff + off);
1334 rval = f(arg, p, count);
1335 if (rval)
1336 return (rval);
1337 len -= count;
1338 off = 0;
1339 } else
1340 off -= pglen;
1341 pgoff = 0;
1342 }
1343 if (len > 0) {
1344 KASSERT(off < m->m_epg_trllen,
1345 ("m_apply_extpg_one: offset beyond trailer"));
1346 KASSERT(len <= m->m_epg_trllen - off,
1347 ("m_apply_extpg_one: length beyond trailer"));
1348 return (f(arg, m->m_epg_trail + off, len));
1349 }
1350 return (0);
1351 }
1352
1353 /* Apply function f to the data in a single mbuf. */
1354 static int
m_apply_one(struct mbuf * m,int off,int len,int (* f)(void *,void *,u_int),void * arg)1355 m_apply_one(struct mbuf *m, int off, int len,
1356 int (*f)(void *, void *, u_int), void *arg)
1357 {
1358 if ((m->m_flags & M_EXTPG) != 0)
1359 return (m_apply_extpg_one(m, off, len, f, arg));
1360 else
1361 return (f(arg, mtod(m, caddr_t) + off, len));
1362 }
1363
1364 /*
1365 * Apply function f to the data in an mbuf chain starting "off" bytes from
1366 * the beginning, continuing for "len" bytes.
1367 */
1368 int
m_apply(struct mbuf * m,int off,int len,int (* f)(void *,void *,u_int),void * arg)1369 m_apply(struct mbuf *m, int off, int len,
1370 int (*f)(void *, void *, u_int), void *arg)
1371 {
1372 u_int count;
1373 int rval;
1374
1375 KASSERT(off >= 0, ("m_apply, negative off %d", off));
1376 KASSERT(len >= 0, ("m_apply, negative len %d", len));
1377 while (off > 0) {
1378 KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
1379 if (off < m->m_len)
1380 break;
1381 off -= m->m_len;
1382 m = m->m_next;
1383 }
1384 while (len > 0) {
1385 KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
1386 count = min(m->m_len - off, len);
1387 rval = m_apply_one(m, off, count, f, arg);
1388 if (rval)
1389 return (rval);
1390 len -= count;
1391 off = 0;
1392 m = m->m_next;
1393 }
1394 return (0);
1395 }
1396
1397 /*
1398 * Return a pointer to mbuf/offset of location in mbuf chain.
1399 */
1400 struct mbuf *
m_getptr(struct mbuf * m,int loc,int * off)1401 m_getptr(struct mbuf *m, int loc, int *off)
1402 {
1403
1404 while (loc >= 0) {
1405 /* Normal end of search. */
1406 if (m->m_len > loc) {
1407 *off = loc;
1408 return (m);
1409 } else {
1410 loc -= m->m_len;
1411 if (m->m_next == NULL) {
1412 if (loc == 0) {
1413 /* Point at the end of valid data. */
1414 *off = m->m_len;
1415 return (m);
1416 }
1417 return (NULL);
1418 }
1419 m = m->m_next;
1420 }
1421 }
1422 return (NULL);
1423 }
1424
1425 void
m_print(const struct mbuf * m,int maxlen)1426 m_print(const struct mbuf *m, int maxlen)
1427 {
1428 int len;
1429 int pdata;
1430 const struct mbuf *m2;
1431
1432 if (m == NULL) {
1433 printf("mbuf: %p\n", m);
1434 return;
1435 }
1436
1437 if (m->m_flags & M_PKTHDR)
1438 len = m->m_pkthdr.len;
1439 else
1440 len = -1;
1441 m2 = m;
1442 while (m2 != NULL && (len == -1 || len)) {
1443 pdata = m2->m_len;
1444 if (maxlen != -1 && pdata > maxlen)
1445 pdata = maxlen;
1446 printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len,
1447 m2->m_next, m2->m_flags, "\20\20freelist\17skipfw"
1448 "\11proto5\10proto4\7proto3\6proto2\5proto1\4rdonly"
1449 "\3eor\2pkthdr\1ext", pdata ? "" : "\n");
1450 if (pdata)
1451 printf(", %*D\n", pdata, (u_char *)m2->m_data, "-");
1452 if (len != -1)
1453 len -= m2->m_len;
1454 m2 = m2->m_next;
1455 }
1456 if (len > 0)
1457 printf("%d bytes unaccounted for.\n", len);
1458 return;
1459 }
1460
1461 u_int
m_fixhdr(struct mbuf * m0)1462 m_fixhdr(struct mbuf *m0)
1463 {
1464 u_int len;
1465
1466 len = m_length(m0, NULL);
1467 m0->m_pkthdr.len = len;
1468 return (len);
1469 }
1470
1471 u_int
m_length(struct mbuf * m0,struct mbuf ** last)1472 m_length(struct mbuf *m0, struct mbuf **last)
1473 {
1474 struct mbuf *m;
1475 u_int len;
1476
1477 len = 0;
1478 for (m = m0; m != NULL; m = m->m_next) {
1479 len += m->m_len;
1480 if (m->m_next == NULL)
1481 break;
1482 }
1483 if (last != NULL)
1484 *last = m;
1485 return (len);
1486 }
1487
1488 /*
1489 * Defragment a mbuf chain, returning the shortest possible
1490 * chain of mbufs and clusters. If allocation fails and
1491 * this cannot be completed, NULL will be returned, but
1492 * the passed in chain will be unchanged. Upon success,
1493 * the original chain will be freed, and the new chain
1494 * will be returned.
1495 *
1496 * If a non-packet header is passed in, the original
1497 * mbuf (chain?) will be returned unharmed.
1498 */
1499 struct mbuf *
m_defrag(struct mbuf * m0,int how)1500 m_defrag(struct mbuf *m0, int how)
1501 {
1502 struct mbuf *m_new = NULL, *m_final = NULL;
1503 int progress = 0, length;
1504
1505 MBUF_CHECKSLEEP(how);
1506 if (!(m0->m_flags & M_PKTHDR))
1507 return (m0);
1508
1509 m_fixhdr(m0); /* Needed sanity check */
1510
1511 #ifdef MBUF_STRESS_TEST
1512 if (m_defragrandomfailures) {
1513 int temp = arc4random() & 0xff;
1514 if (temp == 0xba)
1515 goto nospace;
1516 }
1517 #endif
1518
1519 if (m0->m_pkthdr.len > MHLEN)
1520 m_final = m_getcl(how, MT_DATA, M_PKTHDR);
1521 else
1522 m_final = m_gethdr(how, MT_DATA);
1523
1524 if (m_final == NULL)
1525 goto nospace;
1526
1527 if (m_dup_pkthdr(m_final, m0, how) == 0)
1528 goto nospace;
1529
1530 m_new = m_final;
1531
1532 while (progress < m0->m_pkthdr.len) {
1533 length = m0->m_pkthdr.len - progress;
1534 if (length > MCLBYTES)
1535 length = MCLBYTES;
1536
1537 if (m_new == NULL) {
1538 if (length > MLEN)
1539 m_new = m_getcl(how, MT_DATA, 0);
1540 else
1541 m_new = m_get(how, MT_DATA);
1542 if (m_new == NULL)
1543 goto nospace;
1544 }
1545
1546 m_copydata(m0, progress, length, mtod(m_new, caddr_t));
1547 progress += length;
1548 m_new->m_len = length;
1549 if (m_new != m_final)
1550 m_cat(m_final, m_new);
1551 m_new = NULL;
1552 }
1553 #ifdef MBUF_STRESS_TEST
1554 if (m0->m_next == NULL)
1555 m_defraguseless++;
1556 #endif
1557 m_freem(m0);
1558 m0 = m_final;
1559 #ifdef MBUF_STRESS_TEST
1560 m_defragpackets++;
1561 m_defragbytes += m0->m_pkthdr.len;
1562 #endif
1563 return (m0);
1564 nospace:
1565 #ifdef MBUF_STRESS_TEST
1566 m_defragfailure++;
1567 #endif
1568 if (m_final)
1569 m_freem(m_final);
1570 return (NULL);
1571 }
1572
1573 /*
1574 * Return the number of fragments an mbuf will use. This is usually
1575 * used as a proxy for the number of scatter/gather elements needed by
1576 * a DMA engine to access an mbuf. In general mapped mbufs are
1577 * assumed to be backed by physically contiguous buffers that only
1578 * need a single fragment. Unmapped mbufs, on the other hand, can
1579 * span disjoint physical pages.
1580 */
1581 static int
frags_per_mbuf(struct mbuf * m)1582 frags_per_mbuf(struct mbuf *m)
1583 {
1584 int frags;
1585
1586 if ((m->m_flags & M_EXTPG) == 0)
1587 return (1);
1588
1589 /*
1590 * The header and trailer are counted as a single fragment
1591 * each when present.
1592 *
1593 * XXX: This overestimates the number of fragments by assuming
1594 * all the backing physical pages are disjoint.
1595 */
1596 frags = 0;
1597 if (m->m_epg_hdrlen != 0)
1598 frags++;
1599 frags += m->m_epg_npgs;
1600 if (m->m_epg_trllen != 0)
1601 frags++;
1602
1603 return (frags);
1604 }
1605
1606 /*
1607 * Defragment an mbuf chain, returning at most maxfrags separate
1608 * mbufs+clusters. If this is not possible NULL is returned and
1609 * the original mbuf chain is left in its present (potentially
1610 * modified) state. We use two techniques: collapsing consecutive
1611 * mbufs and replacing consecutive mbufs by a cluster.
1612 *
1613 * NB: this should really be named m_defrag but that name is taken
1614 */
1615 struct mbuf *
m_collapse(struct mbuf * m0,int how,int maxfrags)1616 m_collapse(struct mbuf *m0, int how, int maxfrags)
1617 {
1618 struct mbuf *m, *n, *n2, **prev;
1619 u_int curfrags;
1620
1621 /*
1622 * Calculate the current number of frags.
1623 */
1624 curfrags = 0;
1625 for (m = m0; m != NULL; m = m->m_next)
1626 curfrags += frags_per_mbuf(m);
1627 /*
1628 * First, try to collapse mbufs. Note that we always collapse
1629 * towards the front so we don't need to deal with moving the
1630 * pkthdr. This may be suboptimal if the first mbuf has much
1631 * less data than the following.
1632 */
1633 m = m0;
1634 again:
1635 for (;;) {
1636 n = m->m_next;
1637 if (n == NULL)
1638 break;
1639 if (M_WRITABLE(m) &&
1640 n->m_len < M_TRAILINGSPACE(m)) {
1641 m_copydata(n, 0, n->m_len,
1642 mtod(m, char *) + m->m_len);
1643 m->m_len += n->m_len;
1644 m->m_next = n->m_next;
1645 curfrags -= frags_per_mbuf(n);
1646 m_free(n);
1647 if (curfrags <= maxfrags)
1648 return m0;
1649 } else
1650 m = n;
1651 }
1652 KASSERT(maxfrags > 1,
1653 ("maxfrags %u, but normal collapse failed", maxfrags));
1654 /*
1655 * Collapse consecutive mbufs to a cluster.
1656 */
1657 prev = &m0->m_next; /* NB: not the first mbuf */
1658 while ((n = *prev) != NULL) {
1659 if ((n2 = n->m_next) != NULL &&
1660 n->m_len + n2->m_len < MCLBYTES) {
1661 m = m_getcl(how, MT_DATA, 0);
1662 if (m == NULL)
1663 goto bad;
1664 m_copydata(n, 0, n->m_len, mtod(m, char *));
1665 m_copydata(n2, 0, n2->m_len,
1666 mtod(m, char *) + n->m_len);
1667 m->m_len = n->m_len + n2->m_len;
1668 m->m_next = n2->m_next;
1669 *prev = m;
1670 curfrags += 1; /* For the new cluster */
1671 curfrags -= frags_per_mbuf(n);
1672 curfrags -= frags_per_mbuf(n2);
1673 m_free(n);
1674 m_free(n2);
1675 if (curfrags <= maxfrags)
1676 return m0;
1677 /*
1678 * Still not there, try the normal collapse
1679 * again before we allocate another cluster.
1680 */
1681 goto again;
1682 }
1683 prev = &n->m_next;
1684 }
1685 /*
1686 * No place where we can collapse to a cluster; punt.
1687 * This can occur if, for example, you request 2 frags
1688 * but the packet requires that both be clusters (we
1689 * never reallocate the first mbuf to avoid moving the
1690 * packet header).
1691 */
1692 bad:
1693 return NULL;
1694 }
1695
1696 #ifdef MBUF_STRESS_TEST
1697
1698 /*
1699 * Fragment an mbuf chain. There's no reason you'd ever want to do
1700 * this in normal usage, but it's great for stress testing various
1701 * mbuf consumers.
1702 *
1703 * If fragmentation is not possible, the original chain will be
1704 * returned.
1705 *
1706 * Possible length values:
1707 * 0 no fragmentation will occur
1708 * > 0 each fragment will be of the specified length
1709 * -1 each fragment will be the same random value in length
1710 * -2 each fragment's length will be entirely random
1711 * (Random values range from 1 to 256)
1712 */
1713 struct mbuf *
m_fragment(struct mbuf * m0,int how,int length)1714 m_fragment(struct mbuf *m0, int how, int length)
1715 {
1716 struct mbuf *m_first, *m_last;
1717 int divisor = 255, progress = 0, fraglen;
1718
1719 if (!(m0->m_flags & M_PKTHDR))
1720 return (m0);
1721
1722 if (length == 0 || length < -2)
1723 return (m0);
1724 if (length > MCLBYTES)
1725 length = MCLBYTES;
1726 if (length < 0 && divisor > MCLBYTES)
1727 divisor = MCLBYTES;
1728 if (length == -1)
1729 length = 1 + (arc4random() % divisor);
1730 if (length > 0)
1731 fraglen = length;
1732
1733 m_fixhdr(m0); /* Needed sanity check */
1734
1735 m_first = m_getcl(how, MT_DATA, M_PKTHDR);
1736 if (m_first == NULL)
1737 goto nospace;
1738
1739 if (m_dup_pkthdr(m_first, m0, how) == 0)
1740 goto nospace;
1741
1742 m_last = m_first;
1743
1744 while (progress < m0->m_pkthdr.len) {
1745 if (length == -2)
1746 fraglen = 1 + (arc4random() % divisor);
1747 if (fraglen > m0->m_pkthdr.len - progress)
1748 fraglen = m0->m_pkthdr.len - progress;
1749
1750 if (progress != 0) {
1751 struct mbuf *m_new = m_getcl(how, MT_DATA, 0);
1752 if (m_new == NULL)
1753 goto nospace;
1754
1755 m_last->m_next = m_new;
1756 m_last = m_new;
1757 }
1758
1759 m_copydata(m0, progress, fraglen, mtod(m_last, caddr_t));
1760 progress += fraglen;
1761 m_last->m_len = fraglen;
1762 }
1763 m_freem(m0);
1764 m0 = m_first;
1765 return (m0);
1766 nospace:
1767 if (m_first)
1768 m_freem(m_first);
1769 /* Return the original chain on failure */
1770 return (m0);
1771 }
1772
1773 #endif
1774
1775 /*
1776 * Free pages from mbuf_ext_pgs, assuming they were allocated via
1777 * vm_page_alloc() and aren't associated with any object. Complement
1778 * to allocator from m_uiotombuf_nomap().
1779 */
1780 void
mb_free_mext_pgs(struct mbuf * m)1781 mb_free_mext_pgs(struct mbuf *m)
1782 {
1783 vm_page_t pg;
1784
1785 M_ASSERTEXTPG(m);
1786 for (int i = 0; i < m->m_epg_npgs; i++) {
1787 pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
1788 vm_page_unwire_noq(pg);
1789 vm_page_free(pg);
1790 }
1791 }
1792
1793 static struct mbuf *
m_uiotombuf_nomap(struct uio * uio,int how,int len,int maxseg,int flags)1794 m_uiotombuf_nomap(struct uio *uio, int how, int len, int maxseg, int flags)
1795 {
1796 struct mbuf *m, *mb, *prev;
1797 vm_page_t pg_array[MBUF_PEXT_MAX_PGS];
1798 int error, length, i, needed;
1799 ssize_t total;
1800 int pflags = malloc2vm_flags(how) | VM_ALLOC_NODUMP | VM_ALLOC_WIRED;
1801
1802 MPASS((flags & M_PKTHDR) == 0);
1803 MPASS((how & M_ZERO) == 0);
1804
1805 /*
1806 * len can be zero or an arbitrary large value bound by
1807 * the total data supplied by the uio.
1808 */
1809 if (len > 0)
1810 total = MIN(uio->uio_resid, len);
1811 else
1812 total = uio->uio_resid;
1813
1814 if (maxseg == 0)
1815 maxseg = MBUF_PEXT_MAX_PGS * PAGE_SIZE;
1816
1817 /*
1818 * If total is zero, return an empty mbuf. This can occur
1819 * for TLS 1.0 connections which send empty fragments as
1820 * a countermeasure against the known-IV weakness in CBC
1821 * ciphersuites.
1822 */
1823 if (__predict_false(total == 0)) {
1824 mb = mb_alloc_ext_pgs(how, mb_free_mext_pgs);
1825 if (mb == NULL)
1826 return (NULL);
1827 mb->m_epg_flags = EPG_FLAG_ANON;
1828 return (mb);
1829 }
1830
1831 /*
1832 * Allocate the pages
1833 */
1834 m = NULL;
1835 while (total > 0) {
1836 mb = mb_alloc_ext_pgs(how, mb_free_mext_pgs);
1837 if (mb == NULL)
1838 goto failed;
1839 if (m == NULL)
1840 m = mb;
1841 else
1842 prev->m_next = mb;
1843 prev = mb;
1844 mb->m_epg_flags = EPG_FLAG_ANON;
1845 needed = length = MIN(maxseg, total);
1846 for (i = 0; needed > 0; i++, needed -= PAGE_SIZE) {
1847 retry_page:
1848 pg_array[i] = vm_page_alloc_noobj(pflags);
1849 if (pg_array[i] == NULL) {
1850 if (how & M_NOWAIT) {
1851 goto failed;
1852 } else {
1853 vm_wait(NULL);
1854 goto retry_page;
1855 }
1856 }
1857 mb->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]);
1858 mb->m_epg_npgs++;
1859 }
1860 mb->m_epg_last_len = length - PAGE_SIZE * (mb->m_epg_npgs - 1);
1861 MBUF_EXT_PGS_ASSERT_SANITY(mb);
1862 total -= length;
1863 error = uiomove_fromphys(pg_array, 0, length, uio);
1864 if (error != 0)
1865 goto failed;
1866 mb->m_len = length;
1867 mb->m_ext.ext_size += PAGE_SIZE * mb->m_epg_npgs;
1868 if (flags & M_PKTHDR)
1869 m->m_pkthdr.len += length;
1870 }
1871 return (m);
1872
1873 failed:
1874 m_freem(m);
1875 return (NULL);
1876 }
1877
1878 /*
1879 * Copy the contents of uio into a properly sized mbuf chain.
1880 */
1881 struct mbuf *
m_uiotombuf(struct uio * uio,int how,int len,int align,int flags)1882 m_uiotombuf(struct uio *uio, int how, int len, int align, int flags)
1883 {
1884 struct mbuf *m, *mb;
1885 int error, length;
1886 ssize_t total;
1887 int progress = 0;
1888
1889 if (flags & M_EXTPG)
1890 return (m_uiotombuf_nomap(uio, how, len, align, flags));
1891
1892 /*
1893 * len can be zero or an arbitrary large value bound by
1894 * the total data supplied by the uio.
1895 */
1896 if (len > 0)
1897 total = (uio->uio_resid < len) ? uio->uio_resid : len;
1898 else
1899 total = uio->uio_resid;
1900
1901 /*
1902 * The smallest unit returned by m_getm2() is a single mbuf
1903 * with pkthdr. We can't align past it.
1904 */
1905 if (align >= MHLEN)
1906 return (NULL);
1907
1908 /*
1909 * Give us the full allocation or nothing.
1910 * If len is zero return the smallest empty mbuf.
1911 */
1912 m = m_getm2(NULL, max(total + align, 1), how, MT_DATA, flags);
1913 if (m == NULL)
1914 return (NULL);
1915 m->m_data += align;
1916
1917 /* Fill all mbufs with uio data and update header information. */
1918 for (mb = m; mb != NULL; mb = mb->m_next) {
1919 length = min(M_TRAILINGSPACE(mb), total - progress);
1920
1921 error = uiomove(mtod(mb, void *), length, uio);
1922 if (error) {
1923 m_freem(m);
1924 return (NULL);
1925 }
1926
1927 mb->m_len = length;
1928 progress += length;
1929 if (flags & M_PKTHDR) {
1930 m->m_pkthdr.len += length;
1931 m->m_pkthdr.memlen += MSIZE;
1932 if (mb->m_flags & M_EXT)
1933 m->m_pkthdr.memlen += mb->m_ext.ext_size;
1934 }
1935 }
1936 KASSERT(progress == total, ("%s: progress != total", __func__));
1937
1938 return (m);
1939 }
1940
1941 /*
1942 * Copy data to/from an unmapped mbuf into a uio limited by len if set.
1943 */
1944 int
m_unmapped_uiomove(const struct mbuf * m,int m_off,struct uio * uio,int len)1945 m_unmapped_uiomove(const struct mbuf *m, int m_off, struct uio *uio, int len)
1946 {
1947 vm_page_t pg;
1948 int error, i, off, pglen, pgoff, seglen, segoff;
1949
1950 M_ASSERTEXTPG(m);
1951 error = 0;
1952
1953 /* Skip over any data removed from the front. */
1954 off = mtod(m, vm_offset_t);
1955
1956 off += m_off;
1957 if (m->m_epg_hdrlen != 0) {
1958 if (off >= m->m_epg_hdrlen) {
1959 off -= m->m_epg_hdrlen;
1960 } else {
1961 seglen = m->m_epg_hdrlen - off;
1962 segoff = off;
1963 seglen = min(seglen, len);
1964 off = 0;
1965 len -= seglen;
1966 error = uiomove(__DECONST(void *,
1967 &m->m_epg_hdr[segoff]), seglen, uio);
1968 }
1969 }
1970 pgoff = m->m_epg_1st_off;
1971 for (i = 0; i < m->m_epg_npgs && error == 0 && len > 0; i++) {
1972 pglen = m_epg_pagelen(m, i, pgoff);
1973 if (off >= pglen) {
1974 off -= pglen;
1975 pgoff = 0;
1976 continue;
1977 }
1978 seglen = pglen - off;
1979 segoff = pgoff + off;
1980 off = 0;
1981 seglen = min(seglen, len);
1982 len -= seglen;
1983 pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
1984 error = uiomove_fromphys(&pg, segoff, seglen, uio);
1985 pgoff = 0;
1986 };
1987 if (len != 0 && error == 0) {
1988 KASSERT((off + len) <= m->m_epg_trllen,
1989 ("off + len > trail (%d + %d > %d, m_off = %d)", off, len,
1990 m->m_epg_trllen, m_off));
1991 error = uiomove(__DECONST(void *, &m->m_epg_trail[off]),
1992 len, uio);
1993 }
1994 return (error);
1995 }
1996
1997 /*
1998 * Copy an mbuf chain into a uio limited by len if set.
1999 */
2000 int
m_mbuftouio(struct uio * uio,const struct mbuf * m,int len)2001 m_mbuftouio(struct uio *uio, const struct mbuf *m, int len)
2002 {
2003 int error, length, total;
2004 int progress = 0;
2005
2006 if (len > 0)
2007 total = min(uio->uio_resid, len);
2008 else
2009 total = uio->uio_resid;
2010
2011 /* Fill the uio with data from the mbufs. */
2012 for (; m != NULL; m = m->m_next) {
2013 length = min(m->m_len, total - progress);
2014
2015 if ((m->m_flags & M_EXTPG) != 0)
2016 error = m_unmapped_uiomove(m, 0, uio, length);
2017 else
2018 error = uiomove(mtod(m, void *), length, uio);
2019 if (error)
2020 return (error);
2021
2022 progress += length;
2023 }
2024
2025 return (0);
2026 }
2027
2028 /*
2029 * Create a writable copy of the mbuf chain. While doing this
2030 * we compact the chain with a goal of producing a chain with
2031 * at most two mbufs. The second mbuf in this chain is likely
2032 * to be a cluster. The primary purpose of this work is to create
2033 * a writable packet for encryption, compression, etc. The
2034 * secondary goal is to linearize the data so the data can be
2035 * passed to crypto hardware in the most efficient manner possible.
2036 */
2037 struct mbuf *
m_unshare(struct mbuf * m0,int how)2038 m_unshare(struct mbuf *m0, int how)
2039 {
2040 struct mbuf *m, *mprev;
2041 struct mbuf *n, *mfirst, *mlast;
2042 int len, off;
2043
2044 mprev = NULL;
2045 for (m = m0; m != NULL; m = mprev->m_next) {
2046 /*
2047 * Regular mbufs are ignored unless there's a cluster
2048 * in front of it that we can use to coalesce. We do
2049 * the latter mainly so later clusters can be coalesced
2050 * also w/o having to handle them specially (i.e. convert
2051 * mbuf+cluster -> cluster). This optimization is heavily
2052 * influenced by the assumption that we're running over
2053 * Ethernet where MCLBYTES is large enough that the max
2054 * packet size will permit lots of coalescing into a
2055 * single cluster. This in turn permits efficient
2056 * crypto operations, especially when using hardware.
2057 */
2058 if ((m->m_flags & M_EXT) == 0) {
2059 if (mprev && (mprev->m_flags & M_EXT) &&
2060 m->m_len <= M_TRAILINGSPACE(mprev)) {
2061 /* XXX: this ignores mbuf types */
2062 memcpy(mtod(mprev, caddr_t) + mprev->m_len,
2063 mtod(m, caddr_t), m->m_len);
2064 mprev->m_len += m->m_len;
2065 mprev->m_next = m->m_next; /* unlink from chain */
2066 m_free(m); /* reclaim mbuf */
2067 } else {
2068 mprev = m;
2069 }
2070 continue;
2071 }
2072 /*
2073 * Writable mbufs are left alone (for now).
2074 */
2075 if (M_WRITABLE(m)) {
2076 mprev = m;
2077 continue;
2078 }
2079
2080 /*
2081 * Not writable, replace with a copy or coalesce with
2082 * the previous mbuf if possible (since we have to copy
2083 * it anyway, we try to reduce the number of mbufs and
2084 * clusters so that future work is easier).
2085 */
2086 KASSERT(m->m_flags & M_EXT, ("m_flags 0x%x", m->m_flags));
2087 /* NB: we only coalesce into a cluster or larger */
2088 if (mprev != NULL && (mprev->m_flags & M_EXT) &&
2089 m->m_len <= M_TRAILINGSPACE(mprev)) {
2090 /* XXX: this ignores mbuf types */
2091 memcpy(mtod(mprev, caddr_t) + mprev->m_len,
2092 mtod(m, caddr_t), m->m_len);
2093 mprev->m_len += m->m_len;
2094 mprev->m_next = m->m_next; /* unlink from chain */
2095 m_free(m); /* reclaim mbuf */
2096 continue;
2097 }
2098
2099 /*
2100 * Allocate new space to hold the copy and copy the data.
2101 * We deal with jumbo mbufs (i.e. m_len > MCLBYTES) by
2102 * splitting them into clusters. We could just malloc a
2103 * buffer and make it external but too many device drivers
2104 * don't know how to break up the non-contiguous memory when
2105 * doing DMA.
2106 */
2107 n = m_getcl(how, m->m_type, m->m_flags & M_COPYFLAGS);
2108 if (n == NULL) {
2109 m_freem(m0);
2110 return (NULL);
2111 }
2112 if (m->m_flags & M_PKTHDR) {
2113 KASSERT(mprev == NULL, ("%s: m0 %p, m %p has M_PKTHDR",
2114 __func__, m0, m));
2115 m_move_pkthdr(n, m);
2116 }
2117 len = m->m_len;
2118 off = 0;
2119 mfirst = n;
2120 mlast = NULL;
2121 for (;;) {
2122 int cc = min(len, MCLBYTES);
2123 memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc);
2124 n->m_len = cc;
2125 if (mlast != NULL)
2126 mlast->m_next = n;
2127 mlast = n;
2128 #if 0
2129 newipsecstat.ips_clcopied++;
2130 #endif
2131
2132 len -= cc;
2133 if (len <= 0)
2134 break;
2135 off += cc;
2136
2137 n = m_getcl(how, m->m_type, m->m_flags & M_COPYFLAGS);
2138 if (n == NULL) {
2139 m_freem(mfirst);
2140 m_freem(m0);
2141 return (NULL);
2142 }
2143 }
2144 n->m_next = m->m_next;
2145 if (mprev == NULL)
2146 m0 = mfirst; /* new head of chain */
2147 else
2148 mprev->m_next = mfirst; /* replace old mbuf */
2149 m_free(m); /* release old mbuf */
2150 mprev = mfirst;
2151 }
2152 return (m0);
2153 }
2154
2155 #ifdef MBUF_PROFILING
2156
2157 #define MP_BUCKETS 32 /* don't just change this as things may overflow.*/
2158 struct mbufprofile {
2159 uintmax_t wasted[MP_BUCKETS];
2160 uintmax_t used[MP_BUCKETS];
2161 uintmax_t segments[MP_BUCKETS];
2162 } mbprof;
2163
2164 void
m_profile(struct mbuf * m)2165 m_profile(struct mbuf *m)
2166 {
2167 int segments = 0;
2168 int used = 0;
2169 int wasted = 0;
2170
2171 while (m) {
2172 segments++;
2173 used += m->m_len;
2174 if (m->m_flags & M_EXT) {
2175 wasted += MHLEN - sizeof(m->m_ext) +
2176 m->m_ext.ext_size - m->m_len;
2177 } else {
2178 if (m->m_flags & M_PKTHDR)
2179 wasted += MHLEN - m->m_len;
2180 else
2181 wasted += MLEN - m->m_len;
2182 }
2183 m = m->m_next;
2184 }
2185 /* be paranoid.. it helps */
2186 if (segments > MP_BUCKETS - 1)
2187 segments = MP_BUCKETS - 1;
2188 if (used > 100000)
2189 used = 100000;
2190 if (wasted > 100000)
2191 wasted = 100000;
2192 /* store in the appropriate bucket */
2193 /* don't bother locking. if it's slightly off, so what? */
2194 mbprof.segments[segments]++;
2195 mbprof.used[fls(used)]++;
2196 mbprof.wasted[fls(wasted)]++;
2197 }
2198
2199 static int
mbprof_handler(SYSCTL_HANDLER_ARGS)2200 mbprof_handler(SYSCTL_HANDLER_ARGS)
2201 {
2202 char buf[256];
2203 struct sbuf sb;
2204 int error;
2205 uint64_t *p;
2206
2207 sbuf_new_for_sysctl(&sb, buf, sizeof(buf), req);
2208
2209 p = &mbprof.wasted[0];
2210 sbuf_printf(&sb,
2211 "wasted:\n"
2212 "%ju %ju %ju %ju %ju %ju %ju %ju "
2213 "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2214 p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2215 p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2216 #ifdef BIG_ARRAY
2217 p = &mbprof.wasted[16];
2218 sbuf_printf(&sb,
2219 "%ju %ju %ju %ju %ju %ju %ju %ju "
2220 "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2221 p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2222 p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2223 #endif
2224 p = &mbprof.used[0];
2225 sbuf_printf(&sb,
2226 "used:\n"
2227 "%ju %ju %ju %ju %ju %ju %ju %ju "
2228 "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2229 p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2230 p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2231 #ifdef BIG_ARRAY
2232 p = &mbprof.used[16];
2233 sbuf_printf(&sb,
2234 "%ju %ju %ju %ju %ju %ju %ju %ju "
2235 "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2236 p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2237 p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2238 #endif
2239 p = &mbprof.segments[0];
2240 sbuf_printf(&sb,
2241 "segments:\n"
2242 "%ju %ju %ju %ju %ju %ju %ju %ju "
2243 "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2244 p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2245 p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2246 #ifdef BIG_ARRAY
2247 p = &mbprof.segments[16];
2248 sbuf_printf(&sb,
2249 "%ju %ju %ju %ju %ju %ju %ju %ju "
2250 "%ju %ju %ju %ju %ju %ju %ju %jju",
2251 p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2252 p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2253 #endif
2254
2255 error = sbuf_finish(&sb);
2256 sbuf_delete(&sb);
2257 return (error);
2258 }
2259
2260 static int
mbprof_clr_handler(SYSCTL_HANDLER_ARGS)2261 mbprof_clr_handler(SYSCTL_HANDLER_ARGS)
2262 {
2263 int clear, error;
2264
2265 clear = 0;
2266 error = sysctl_handle_int(oidp, &clear, 0, req);
2267 if (error || !req->newptr)
2268 return (error);
2269
2270 if (clear) {
2271 bzero(&mbprof, sizeof(mbprof));
2272 }
2273
2274 return (error);
2275 }
2276
2277 SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofile,
2278 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
2279 mbprof_handler, "A",
2280 "mbuf profiling statistics");
2281
2282 SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofileclr,
2283 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
2284 mbprof_clr_handler, "I",
2285 "clear mbuf profiling statistics");
2286 #endif
2287