1 /* $OpenBSD: criov.c,v 1.9 2002/01/29 15:48:29 jason Exp $ */
2
3 /*-
4 * Copyright (c) 1999 Theo de Raadt
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 * derived from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/proc.h>
36 #include <sys/errno.h>
37 #include <sys/malloc.h>
38 #include <sys/kernel.h>
39 #include <sys/mbuf.h>
40 #include <sys/uio.h>
41 #include <sys/limits.h>
42 #include <sys/lock.h>
43 #include <sys/sdt.h>
44
45 #include <machine/vmparam.h>
46
47 #include <vm/vm.h>
48 #include <vm/vm_page.h>
49 #include <vm/pmap.h>
50
51 #include <opencrypto/cryptodev.h>
52
53 SDT_PROVIDER_DECLARE(opencrypto);
54
55 /*
56 * These macros are only for avoiding code duplication, as we need to skip
57 * given number of bytes in the same way in several functions below.
58 */
59 #define CUIO_SKIP() do { \
60 KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
61 KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \
62 while (off > 0) { \
63 KASSERT(iol >= 0, ("%s: empty in skip", __func__)); \
64 if (off < iov->iov_len) \
65 break; \
66 off -= iov->iov_len; \
67 iol--; \
68 iov++; \
69 } \
70 } while (0)
71
72 #define CVM_PAGE_SKIP() do { \
73 KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
74 KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \
75 while (off > 0) { \
76 if (off < PAGE_SIZE) \
77 break; \
78 processed += PAGE_SIZE - off; \
79 off -= PAGE_SIZE - off; \
80 pages++; \
81 } \
82 } while (0)
83
84 static void
cuio_copydata(struct uio * uio,int off,int len,caddr_t cp)85 cuio_copydata(struct uio* uio, int off, int len, caddr_t cp)
86 {
87 struct iovec *iov = uio->uio_iov;
88 int iol = uio->uio_iovcnt;
89 unsigned count;
90
91 CUIO_SKIP();
92 while (len > 0) {
93 KASSERT(iol >= 0, ("%s: empty", __func__));
94 count = min(iov->iov_len - off, len);
95 bcopy(((caddr_t)iov->iov_base) + off, cp, count);
96 len -= count;
97 cp += count;
98 off = 0;
99 iol--;
100 iov++;
101 }
102 }
103
104 static void
cuio_copyback(struct uio * uio,int off,int len,c_caddr_t cp)105 cuio_copyback(struct uio* uio, int off, int len, c_caddr_t cp)
106 {
107 struct iovec *iov = uio->uio_iov;
108 int iol = uio->uio_iovcnt;
109 unsigned count;
110
111 CUIO_SKIP();
112 while (len > 0) {
113 KASSERT(iol >= 0, ("%s: empty", __func__));
114 count = min(iov->iov_len - off, len);
115 bcopy(cp, ((caddr_t)iov->iov_base) + off, count);
116 len -= count;
117 cp += count;
118 off = 0;
119 iol--;
120 iov++;
121 }
122 }
123
124 /*
125 * Return the index and offset of location in iovec list.
126 */
127 static int
cuio_getptr(struct uio * uio,int loc,int * off)128 cuio_getptr(struct uio *uio, int loc, int *off)
129 {
130 int ind, len;
131
132 ind = 0;
133 while (loc >= 0 && ind < uio->uio_iovcnt) {
134 len = uio->uio_iov[ind].iov_len;
135 if (len > loc) {
136 *off = loc;
137 return (ind);
138 }
139 loc -= len;
140 ind++;
141 }
142
143 if (ind > 0 && loc == 0) {
144 ind--;
145 *off = uio->uio_iov[ind].iov_len;
146 return (ind);
147 }
148
149 return (-1);
150 }
151
152 #if CRYPTO_MAY_HAVE_VMPAGE
153 /*
154 * Apply function f to the data in a vm_page_t list starting "off" bytes from
155 * the beginning, continuing for "len" bytes.
156 */
157 static int
cvm_page_apply(vm_page_t * pages,int off,int len,int (* f)(void *,const void *,u_int),void * arg)158 cvm_page_apply(vm_page_t *pages, int off, int len,
159 int (*f)(void *, const void *, u_int), void *arg)
160 {
161 int processed = 0;
162 unsigned count;
163 int rval;
164
165 CVM_PAGE_SKIP();
166 while (len > 0) {
167 char *kaddr = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages));
168 count = min(PAGE_SIZE - off, len);
169 rval = (*f)(arg, kaddr + off, count);
170 if (rval)
171 return (rval);
172 len -= count;
173 processed += count;
174 off = 0;
175 pages++;
176 }
177 return (0);
178 }
179
180 static inline void *
cvm_page_contiguous_segment(vm_page_t * pages,size_t skip,int len)181 cvm_page_contiguous_segment(vm_page_t *pages, size_t skip, int len)
182 {
183 if ((skip + len - 1) / PAGE_SIZE > skip / PAGE_SIZE)
184 return (NULL);
185
186 pages += (skip / PAGE_SIZE);
187 skip -= rounddown(skip, PAGE_SIZE);
188 return (((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages))) + skip);
189 }
190
191 /*
192 * Copy len bytes of data from the vm_page_t array, skipping the first off
193 * bytes, into the pointer cp. Return the number of bytes skipped and copied.
194 * Does not verify the length of the array.
195 */
196 static int
cvm_page_copyback(vm_page_t * pages,int off,int len,c_caddr_t cp)197 cvm_page_copyback(vm_page_t *pages, int off, int len, c_caddr_t cp)
198 {
199 int processed = 0;
200 unsigned count;
201
202 CVM_PAGE_SKIP();
203 while (len > 0) {
204 count = min(PAGE_SIZE - off, len);
205 bcopy(cp, (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off,
206 count);
207 len -= count;
208 cp += count;
209 processed += count;
210 off = 0;
211 pages++;
212 }
213 return (processed);
214 }
215
216 /*
217 * Copy len bytes of data from the pointer cp into the vm_page_t array,
218 * skipping the first off bytes, Return the number of bytes skipped and copied.
219 * Does not verify the length of the array.
220 */
221 static int
cvm_page_copydata(vm_page_t * pages,int off,int len,caddr_t cp)222 cvm_page_copydata(vm_page_t *pages, int off, int len, caddr_t cp)
223 {
224 int processed = 0;
225 unsigned count;
226
227 CVM_PAGE_SKIP();
228 while (len > 0) {
229 count = min(PAGE_SIZE - off, len);
230 bcopy(((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off), cp,
231 count);
232 len -= count;
233 cp += count;
234 processed += count;
235 off = 0;
236 pages++;
237 }
238 return processed;
239 }
240 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
241
242 /*
243 * Given a starting page in an m_epg, determine the length of the
244 * current physically contiguous segment.
245 */
246 static __inline size_t
m_epg_pages_extent(struct mbuf * m,int idx,u_int pglen)247 m_epg_pages_extent(struct mbuf *m, int idx, u_int pglen)
248 {
249 size_t len;
250 u_int i;
251
252 len = pglen;
253 for (i = idx + 1; i < m->m_epg_npgs; i++) {
254 if (m->m_epg_pa[i - 1] + PAGE_SIZE != m->m_epg_pa[i])
255 break;
256 len += m_epg_pagelen(m, i, 0);
257 }
258 return (len);
259 }
260
261 static void *
m_epg_segment(struct mbuf * m,size_t offset,size_t * len)262 m_epg_segment(struct mbuf *m, size_t offset, size_t *len)
263 {
264 u_int i, pglen, pgoff;
265
266 offset += mtod(m, vm_offset_t);
267 if (offset < m->m_epg_hdrlen) {
268 *len = m->m_epg_hdrlen - offset;
269 return (m->m_epg_hdr + offset);
270 }
271 offset -= m->m_epg_hdrlen;
272 pgoff = m->m_epg_1st_off;
273 for (i = 0; i < m->m_epg_npgs; i++) {
274 pglen = m_epg_pagelen(m, i, pgoff);
275 if (offset < pglen) {
276 *len = m_epg_pages_extent(m, i, pglen) - offset;
277 return ((void *)PHYS_TO_DMAP(m->m_epg_pa[i] + pgoff +
278 offset));
279 }
280 offset -= pglen;
281 pgoff = 0;
282 }
283 KASSERT(offset <= m->m_epg_trllen, ("%s: offset beyond trailer",
284 __func__));
285 *len = m->m_epg_trllen - offset;
286 return (m->m_epg_trail + offset);
287 }
288
289 static __inline void *
m_epg_contiguous_subsegment(struct mbuf * m,size_t skip,size_t len)290 m_epg_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
291 {
292 void *base;
293 size_t seglen;
294
295 base = m_epg_segment(m, skip, &seglen);
296 if (len > seglen)
297 return (NULL);
298 return (base);
299 }
300
301 void
crypto_cursor_init(struct crypto_buffer_cursor * cc,const struct crypto_buffer * cb)302 crypto_cursor_init(struct crypto_buffer_cursor *cc,
303 const struct crypto_buffer *cb)
304 {
305 memset(cc, 0, sizeof(*cc));
306 cc->cc_type = cb->cb_type;
307 switch (cc->cc_type) {
308 case CRYPTO_BUF_CONTIG:
309 cc->cc_buf = cb->cb_buf;
310 cc->cc_buf_len = cb->cb_buf_len;
311 break;
312 case CRYPTO_BUF_MBUF:
313 case CRYPTO_BUF_SINGLE_MBUF:
314 cc->cc_mbuf = cb->cb_mbuf;
315 break;
316 case CRYPTO_BUF_VMPAGE:
317 cc->cc_vmpage = cb->cb_vm_page;
318 cc->cc_buf_len = cb->cb_vm_page_len;
319 cc->cc_offset = cb->cb_vm_page_offset;
320 break;
321 case CRYPTO_BUF_UIO:
322 cc->cc_iov = cb->cb_uio->uio_iov;
323 break;
324 default:
325 #ifdef INVARIANTS
326 panic("%s: invalid buffer type %d", __func__, cb->cb_type);
327 #endif
328 break;
329 }
330 }
331
332 SDT_PROBE_DEFINE2(opencrypto, criov, cursor_advance, vmpage, "struct crypto_buffer_cursor*", "size_t");
333
334 void
crypto_cursor_advance(struct crypto_buffer_cursor * cc,size_t amount)335 crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
336 {
337 size_t remain;
338
339 switch (cc->cc_type) {
340 case CRYPTO_BUF_CONTIG:
341 MPASS(cc->cc_buf_len >= amount);
342 cc->cc_buf += amount;
343 cc->cc_buf_len -= amount;
344 break;
345 case CRYPTO_BUF_MBUF:
346 for (;;) {
347 remain = cc->cc_mbuf->m_len - cc->cc_offset;
348 if (amount < remain) {
349 cc->cc_offset += amount;
350 break;
351 }
352 amount -= remain;
353 cc->cc_mbuf = cc->cc_mbuf->m_next;
354 cc->cc_offset = 0;
355 if (amount == 0)
356 break;
357 }
358 break;
359 case CRYPTO_BUF_SINGLE_MBUF:
360 MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + amount);
361 cc->cc_offset += amount;
362 break;
363 case CRYPTO_BUF_VMPAGE:
364 for (;;) {
365 SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage,
366 cc, amount);
367 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
368 if (amount < remain) {
369 cc->cc_buf_len -= amount;
370 cc->cc_offset += amount;
371 break;
372 }
373 cc->cc_buf_len -= remain;
374 amount -= remain;
375 cc->cc_vmpage++;
376 cc->cc_offset = 0;
377 if (amount == 0 || cc->cc_buf_len == 0)
378 break;
379 }
380 break;
381 case CRYPTO_BUF_UIO:
382 for (;;) {
383 remain = cc->cc_iov->iov_len - cc->cc_offset;
384 if (amount < remain) {
385 cc->cc_offset += amount;
386 break;
387 }
388 amount -= remain;
389 cc->cc_iov++;
390 cc->cc_offset = 0;
391 if (amount == 0)
392 break;
393 }
394 break;
395 default:
396 #ifdef INVARIANTS
397 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
398 #endif
399 break;
400 }
401 }
402
403 void *
crypto_cursor_segment(struct crypto_buffer_cursor * cc,size_t * len)404 crypto_cursor_segment(struct crypto_buffer_cursor *cc, size_t *len)
405 {
406 switch (cc->cc_type) {
407 case CRYPTO_BUF_CONTIG:
408 *len = cc->cc_buf_len;
409 return (cc->cc_buf);
410 case CRYPTO_BUF_MBUF:
411 case CRYPTO_BUF_SINGLE_MBUF:
412 if (cc->cc_mbuf == NULL) {
413 *len = 0;
414 return (NULL);
415 }
416 if (cc->cc_mbuf->m_flags & M_EXTPG)
417 return (m_epg_segment(cc->cc_mbuf, cc->cc_offset, len));
418 *len = cc->cc_mbuf->m_len - cc->cc_offset;
419 return (mtod(cc->cc_mbuf, char *) + cc->cc_offset);
420 case CRYPTO_BUF_VMPAGE:
421 *len = PAGE_SIZE - cc->cc_offset;
422 return ((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
423 *cc->cc_vmpage)) + cc->cc_offset);
424 case CRYPTO_BUF_UIO:
425 *len = cc->cc_iov->iov_len - cc->cc_offset;
426 return ((char *)cc->cc_iov->iov_base + cc->cc_offset);
427 default:
428 #ifdef INVARIANTS
429 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
430 #endif
431 *len = 0;
432 return (NULL);
433 }
434 }
435
436 void *
crypto_cursor_segbase(struct crypto_buffer_cursor * cc)437 crypto_cursor_segbase(struct crypto_buffer_cursor *cc)
438 {
439 size_t len;
440
441 return (crypto_cursor_segment(cc, &len));
442 }
443
444 size_t
crypto_cursor_seglen(struct crypto_buffer_cursor * cc)445 crypto_cursor_seglen(struct crypto_buffer_cursor *cc)
446 {
447 size_t len;
448
449 crypto_cursor_segment(cc, &len);
450 return (len);
451 }
452
453 void
crypto_cursor_copyback(struct crypto_buffer_cursor * cc,int size,const void * vsrc)454 crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size,
455 const void *vsrc)
456 {
457 size_t remain, todo;
458 const char *src;
459 char *dst;
460
461 src = vsrc;
462 switch (cc->cc_type) {
463 case CRYPTO_BUF_CONTIG:
464 MPASS(cc->cc_buf_len >= size);
465 memcpy(cc->cc_buf, src, size);
466 cc->cc_buf += size;
467 cc->cc_buf_len -= size;
468 break;
469 case CRYPTO_BUF_MBUF:
470 for (;;) {
471 /*
472 * This uses m_copyback() for individual
473 * mbufs so that cc_mbuf and cc_offset are
474 * updated.
475 */
476 remain = cc->cc_mbuf->m_len - cc->cc_offset;
477 todo = MIN(remain, size);
478 m_copyback(cc->cc_mbuf, cc->cc_offset, todo, src);
479 src += todo;
480 if (todo < remain) {
481 cc->cc_offset += todo;
482 break;
483 }
484 size -= todo;
485 cc->cc_mbuf = cc->cc_mbuf->m_next;
486 cc->cc_offset = 0;
487 if (size == 0)
488 break;
489 }
490 break;
491 case CRYPTO_BUF_SINGLE_MBUF:
492 MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
493 m_copyback(cc->cc_mbuf, cc->cc_offset, size, src);
494 cc->cc_offset += size;
495 break;
496 case CRYPTO_BUF_VMPAGE:
497 for (;;) {
498 dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
499 *cc->cc_vmpage)) + cc->cc_offset;
500 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
501 todo = MIN(remain, size);
502 memcpy(dst, src, todo);
503 src += todo;
504 cc->cc_buf_len -= todo;
505 if (todo < remain) {
506 cc->cc_offset += todo;
507 break;
508 }
509 size -= todo;
510 cc->cc_vmpage++;
511 cc->cc_offset = 0;
512 if (size == 0)
513 break;
514 }
515 break;
516 case CRYPTO_BUF_UIO:
517 for (;;) {
518 dst = (char *)cc->cc_iov->iov_base + cc->cc_offset;
519 remain = cc->cc_iov->iov_len - cc->cc_offset;
520 todo = MIN(remain, size);
521 memcpy(dst, src, todo);
522 src += todo;
523 if (todo < remain) {
524 cc->cc_offset += todo;
525 break;
526 }
527 size -= todo;
528 cc->cc_iov++;
529 cc->cc_offset = 0;
530 if (size == 0)
531 break;
532 }
533 break;
534 default:
535 #ifdef INVARIANTS
536 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
537 #endif
538 break;
539 }
540 }
541
542 void
crypto_cursor_copydata(struct crypto_buffer_cursor * cc,int size,void * vdst)543 crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst)
544 {
545 size_t remain, todo;
546 const char *src;
547 char *dst;
548
549 dst = vdst;
550 switch (cc->cc_type) {
551 case CRYPTO_BUF_CONTIG:
552 MPASS(cc->cc_buf_len >= size);
553 memcpy(dst, cc->cc_buf, size);
554 cc->cc_buf += size;
555 cc->cc_buf_len -= size;
556 break;
557 case CRYPTO_BUF_MBUF:
558 for (;;) {
559 /*
560 * This uses m_copydata() for individual
561 * mbufs so that cc_mbuf and cc_offset are
562 * updated.
563 */
564 remain = cc->cc_mbuf->m_len - cc->cc_offset;
565 todo = MIN(remain, size);
566 m_copydata(cc->cc_mbuf, cc->cc_offset, todo, dst);
567 dst += todo;
568 if (todo < remain) {
569 cc->cc_offset += todo;
570 break;
571 }
572 size -= todo;
573 cc->cc_mbuf = cc->cc_mbuf->m_next;
574 cc->cc_offset = 0;
575 if (size == 0)
576 break;
577 }
578 break;
579 case CRYPTO_BUF_SINGLE_MBUF:
580 MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
581 m_copydata(cc->cc_mbuf, cc->cc_offset, size, dst);
582 cc->cc_offset += size;
583 break;
584 case CRYPTO_BUF_VMPAGE:
585 for (;;) {
586 src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
587 *cc->cc_vmpage)) + cc->cc_offset;
588 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
589 todo = MIN(remain, size);
590 memcpy(dst, src, todo);
591 src += todo;
592 cc->cc_buf_len -= todo;
593 if (todo < remain) {
594 cc->cc_offset += todo;
595 break;
596 }
597 size -= todo;
598 cc->cc_vmpage++;
599 cc->cc_offset = 0;
600 if (size == 0)
601 break;
602 }
603 break;
604 case CRYPTO_BUF_UIO:
605 for (;;) {
606 src = (const char *)cc->cc_iov->iov_base +
607 cc->cc_offset;
608 remain = cc->cc_iov->iov_len - cc->cc_offset;
609 todo = MIN(remain, size);
610 memcpy(dst, src, todo);
611 dst += todo;
612 if (todo < remain) {
613 cc->cc_offset += todo;
614 break;
615 }
616 size -= todo;
617 cc->cc_iov++;
618 cc->cc_offset = 0;
619 if (size == 0)
620 break;
621 }
622 break;
623 default:
624 #ifdef INVARIANTS
625 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
626 #endif
627 break;
628 }
629 }
630
631 /*
632 * To avoid advancing 'cursor', make a local copy that gets advanced
633 * instead.
634 */
635 void
crypto_cursor_copydata_noadv(struct crypto_buffer_cursor * cc,int size,void * vdst)636 crypto_cursor_copydata_noadv(struct crypto_buffer_cursor *cc, int size,
637 void *vdst)
638 {
639 struct crypto_buffer_cursor copy;
640
641 copy = *cc;
642 crypto_cursor_copydata(©, size, vdst);
643 }
644
645 /*
646 * Apply function f to the data in an iovec list starting "off" bytes from
647 * the beginning, continuing for "len" bytes.
648 */
649 static int
cuio_apply(struct uio * uio,int off,int len,int (* f)(void *,const void *,u_int),void * arg)650 cuio_apply(struct uio *uio, int off, int len,
651 int (*f)(void *, const void *, u_int), void *arg)
652 {
653 struct iovec *iov = uio->uio_iov;
654 int iol = uio->uio_iovcnt;
655 unsigned count;
656 int rval;
657
658 CUIO_SKIP();
659 while (len > 0) {
660 KASSERT(iol >= 0, ("%s: empty", __func__));
661 count = min(iov->iov_len - off, len);
662 rval = (*f)(arg, ((caddr_t)iov->iov_base) + off, count);
663 if (rval)
664 return (rval);
665 len -= count;
666 off = 0;
667 iol--;
668 iov++;
669 }
670 return (0);
671 }
672
673 void
crypto_copyback(struct cryptop * crp,int off,int size,const void * src)674 crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
675 {
676 struct crypto_buffer *cb;
677
678 if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE)
679 cb = &crp->crp_obuf;
680 else
681 cb = &crp->crp_buf;
682 switch (cb->cb_type) {
683 case CRYPTO_BUF_MBUF:
684 case CRYPTO_BUF_SINGLE_MBUF:
685 m_copyback(cb->cb_mbuf, off, size, src);
686 break;
687 #if CRYPTO_MAY_HAVE_VMPAGE
688 case CRYPTO_BUF_VMPAGE:
689 MPASS(size <= cb->cb_vm_page_len);
690 MPASS(size + off <=
691 cb->cb_vm_page_len + cb->cb_vm_page_offset);
692 cvm_page_copyback(cb->cb_vm_page,
693 off + cb->cb_vm_page_offset, size, src);
694 break;
695 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
696 case CRYPTO_BUF_UIO:
697 cuio_copyback(cb->cb_uio, off, size, src);
698 break;
699 case CRYPTO_BUF_CONTIG:
700 MPASS(off + size <= cb->cb_buf_len);
701 bcopy(src, cb->cb_buf + off, size);
702 break;
703 default:
704 #ifdef INVARIANTS
705 panic("invalid crp buf type %d", cb->cb_type);
706 #endif
707 break;
708 }
709 }
710
711 void
crypto_copydata(struct cryptop * crp,int off,int size,void * dst)712 crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
713 {
714
715 switch (crp->crp_buf.cb_type) {
716 case CRYPTO_BUF_MBUF:
717 case CRYPTO_BUF_SINGLE_MBUF:
718 m_copydata(crp->crp_buf.cb_mbuf, off, size, dst);
719 break;
720 #if CRYPTO_MAY_HAVE_VMPAGE
721 case CRYPTO_BUF_VMPAGE:
722 MPASS(size <= crp->crp_buf.cb_vm_page_len);
723 MPASS(size + off <= crp->crp_buf.cb_vm_page_len +
724 crp->crp_buf.cb_vm_page_offset);
725 cvm_page_copydata(crp->crp_buf.cb_vm_page,
726 off + crp->crp_buf.cb_vm_page_offset, size, dst);
727 break;
728 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
729 case CRYPTO_BUF_UIO:
730 cuio_copydata(crp->crp_buf.cb_uio, off, size, dst);
731 break;
732 case CRYPTO_BUF_CONTIG:
733 MPASS(off + size <= crp->crp_buf.cb_buf_len);
734 bcopy(crp->crp_buf.cb_buf + off, dst, size);
735 break;
736 default:
737 #ifdef INVARIANTS
738 panic("invalid crp buf type %d", crp->crp_buf.cb_type);
739 #endif
740 break;
741 }
742 }
743
744 int
crypto_apply_buf(struct crypto_buffer * cb,int off,int len,int (* f)(void *,const void *,u_int),void * arg)745 crypto_apply_buf(struct crypto_buffer *cb, int off, int len,
746 int (*f)(void *, const void *, u_int), void *arg)
747 {
748 int error;
749
750 switch (cb->cb_type) {
751 case CRYPTO_BUF_MBUF:
752 case CRYPTO_BUF_SINGLE_MBUF:
753 error = m_apply(cb->cb_mbuf, off, len,
754 (int (*)(void *, void *, u_int))f, arg);
755 break;
756 case CRYPTO_BUF_UIO:
757 error = cuio_apply(cb->cb_uio, off, len, f, arg);
758 break;
759 #if CRYPTO_MAY_HAVE_VMPAGE
760 case CRYPTO_BUF_VMPAGE:
761 error = cvm_page_apply(cb->cb_vm_page,
762 off + cb->cb_vm_page_offset, len, f, arg);
763 break;
764 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
765 case CRYPTO_BUF_CONTIG:
766 MPASS(off + len <= cb->cb_buf_len);
767 error = (*f)(arg, cb->cb_buf + off, len);
768 break;
769 default:
770 #ifdef INVARIANTS
771 panic("invalid crypto buf type %d", cb->cb_type);
772 #endif
773 error = 0;
774 break;
775 }
776 return (error);
777 }
778
779 int
crypto_apply(struct cryptop * crp,int off,int len,int (* f)(void *,const void *,u_int),void * arg)780 crypto_apply(struct cryptop *crp, int off, int len,
781 int (*f)(void *, const void *, u_int), void *arg)
782 {
783 return (crypto_apply_buf(&crp->crp_buf, off, len, f, arg));
784 }
785
786 static inline void *
m_contiguous_subsegment(struct mbuf * m,size_t skip,size_t len)787 m_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
788 {
789 int rel_off;
790
791 MPASS(skip <= INT_MAX);
792
793 m = m_getptr(m, (int)skip, &rel_off);
794 if (m == NULL)
795 return (NULL);
796
797 MPASS(rel_off >= 0);
798 skip = rel_off;
799 if (skip + len > m->m_len)
800 return (NULL);
801
802 if (m->m_flags & M_EXTPG)
803 return (m_epg_contiguous_subsegment(m, skip, len));
804 return (mtod(m, char*) + skip);
805 }
806
807 static inline void *
cuio_contiguous_segment(struct uio * uio,size_t skip,size_t len)808 cuio_contiguous_segment(struct uio *uio, size_t skip, size_t len)
809 {
810 int rel_off, idx;
811
812 MPASS(skip <= INT_MAX);
813 idx = cuio_getptr(uio, (int)skip, &rel_off);
814 if (idx < 0)
815 return (NULL);
816
817 MPASS(rel_off >= 0);
818 skip = rel_off;
819 if (skip + len > uio->uio_iov[idx].iov_len)
820 return (NULL);
821 return ((char *)uio->uio_iov[idx].iov_base + skip);
822 }
823
824 void *
crypto_buffer_contiguous_subsegment(struct crypto_buffer * cb,size_t skip,size_t len)825 crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip,
826 size_t len)
827 {
828
829 switch (cb->cb_type) {
830 case CRYPTO_BUF_MBUF:
831 case CRYPTO_BUF_SINGLE_MBUF:
832 return (m_contiguous_subsegment(cb->cb_mbuf, skip, len));
833 case CRYPTO_BUF_UIO:
834 return (cuio_contiguous_segment(cb->cb_uio, skip, len));
835 #if CRYPTO_MAY_HAVE_VMPAGE
836 case CRYPTO_BUF_VMPAGE:
837 MPASS(skip + len <= cb->cb_vm_page_len);
838 return (cvm_page_contiguous_segment(cb->cb_vm_page,
839 skip + cb->cb_vm_page_offset, len));
840 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
841 case CRYPTO_BUF_CONTIG:
842 MPASS(skip + len <= cb->cb_buf_len);
843 return (cb->cb_buf + skip);
844 default:
845 #ifdef INVARIANTS
846 panic("invalid crp buf type %d", cb->cb_type);
847 #endif
848 return (NULL);
849 }
850 }
851
852 void *
crypto_contiguous_subsegment(struct cryptop * crp,size_t skip,size_t len)853 crypto_contiguous_subsegment(struct cryptop *crp, size_t skip, size_t len)
854 {
855 return (crypto_buffer_contiguous_subsegment(&crp->crp_buf, skip, len));
856 }
857