1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1991, 1993
5 * The Regents of the University of California. All rights reserved.
6 * (c) UNIX System Laboratories, Inc.
7 * All or some portions of this file are derived from material licensed
8 * to the University of California by American Telephone and Telegraph
9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
10 * the permission of UNIX System Laboratories, Inc.
11 *
12 * Copyright (c) 2014 The FreeBSD Foundation
13 *
14 * Portions of this software were developed by Konstantin Belousov
15 * under sponsorship from the FreeBSD Foundation.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 * 1. Redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution.
25 * 3. Neither the name of the University nor the names of its contributors
26 * may be used to endorse or promote products derived from this software
27 * without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 * SUCH DAMAGE.
40 *
41 * @(#)kern_subr.c 8.3 (Berkeley) 1/21/94
42 */
43
44 #include <sys/cdefs.h>
45 #include <sys/param.h>
46 #include <sys/systm.h>
47 #include <sys/kernel.h>
48 #include <sys/limits.h>
49 #include <sys/lock.h>
50 #include <sys/mman.h>
51 #include <sys/proc.h>
52 #include <sys/resourcevar.h>
53 #include <sys/rwlock.h>
54 #include <sys/sched.h>
55 #include <sys/sysctl.h>
56 #include <sys/vnode.h>
57
58 #include <vm/vm.h>
59 #include <vm/vm_param.h>
60 #include <vm/vm_extern.h>
61 #include <vm/vm_page.h>
62 #include <vm/vm_pageout.h>
63 #include <vm/vm_map.h>
64
65 #include <machine/bus.h>
66
67 SYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, UIO_MAXIOV,
68 "Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)");
69
70 static int uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault);
71
72 int
copyin_nofault(const void * udaddr,void * kaddr,size_t len)73 copyin_nofault(const void *udaddr, void *kaddr, size_t len)
74 {
75 int error, save;
76
77 save = vm_fault_disable_pagefaults();
78 error = copyin(udaddr, kaddr, len);
79 vm_fault_enable_pagefaults(save);
80 return (error);
81 }
82
83 int
copyout_nofault(const void * kaddr,void * udaddr,size_t len)84 copyout_nofault(const void *kaddr, void *udaddr, size_t len)
85 {
86 int error, save;
87
88 save = vm_fault_disable_pagefaults();
89 error = copyout(kaddr, udaddr, len);
90 vm_fault_enable_pagefaults(save);
91 return (error);
92 }
93
94 #define PHYS_PAGE_COUNT(len) (howmany(len, PAGE_SIZE) + 1)
95
96 int
physcopyin(void * src,vm_paddr_t dst,size_t len)97 physcopyin(void *src, vm_paddr_t dst, size_t len)
98 {
99 vm_page_t m[PHYS_PAGE_COUNT(len)];
100 struct iovec iov[1];
101 struct uio uio;
102 int i;
103
104 iov[0].iov_base = src;
105 iov[0].iov_len = len;
106 uio.uio_iov = iov;
107 uio.uio_iovcnt = 1;
108 uio.uio_offset = 0;
109 uio.uio_resid = len;
110 uio.uio_segflg = UIO_SYSSPACE;
111 uio.uio_rw = UIO_WRITE;
112 for (i = 0; i < PHYS_PAGE_COUNT(len); i++, dst += PAGE_SIZE)
113 m[i] = PHYS_TO_VM_PAGE(dst);
114 return (uiomove_fromphys(m, dst & PAGE_MASK, len, &uio));
115 }
116
117 int
physcopyout(vm_paddr_t src,void * dst,size_t len)118 physcopyout(vm_paddr_t src, void *dst, size_t len)
119 {
120 vm_page_t m[PHYS_PAGE_COUNT(len)];
121 struct iovec iov[1];
122 struct uio uio;
123 int i;
124
125 iov[0].iov_base = dst;
126 iov[0].iov_len = len;
127 uio.uio_iov = iov;
128 uio.uio_iovcnt = 1;
129 uio.uio_offset = 0;
130 uio.uio_resid = len;
131 uio.uio_segflg = UIO_SYSSPACE;
132 uio.uio_rw = UIO_READ;
133 for (i = 0; i < PHYS_PAGE_COUNT(len); i++, src += PAGE_SIZE)
134 m[i] = PHYS_TO_VM_PAGE(src);
135 return (uiomove_fromphys(m, src & PAGE_MASK, len, &uio));
136 }
137
138 #undef PHYS_PAGE_COUNT
139
140 int
physcopyin_vlist(bus_dma_segment_t * src,off_t offset,vm_paddr_t dst,size_t len)141 physcopyin_vlist(bus_dma_segment_t *src, off_t offset, vm_paddr_t dst,
142 size_t len)
143 {
144 size_t seg_len;
145 int error;
146
147 error = 0;
148 while (offset >= src->ds_len) {
149 offset -= src->ds_len;
150 src++;
151 }
152
153 while (len > 0 && error == 0) {
154 seg_len = MIN(src->ds_len - offset, len);
155 error = physcopyin((void *)(uintptr_t)(src->ds_addr + offset),
156 dst, seg_len);
157 offset = 0;
158 src++;
159 len -= seg_len;
160 dst += seg_len;
161 }
162
163 return (error);
164 }
165
166 int
physcopyout_vlist(vm_paddr_t src,bus_dma_segment_t * dst,off_t offset,size_t len)167 physcopyout_vlist(vm_paddr_t src, bus_dma_segment_t *dst, off_t offset,
168 size_t len)
169 {
170 size_t seg_len;
171 int error;
172
173 error = 0;
174 while (offset >= dst->ds_len) {
175 offset -= dst->ds_len;
176 dst++;
177 }
178
179 while (len > 0 && error == 0) {
180 seg_len = MIN(dst->ds_len - offset, len);
181 error = physcopyout(src, (void *)(uintptr_t)(dst->ds_addr +
182 offset), seg_len);
183 offset = 0;
184 dst++;
185 len -= seg_len;
186 src += seg_len;
187 }
188
189 return (error);
190 }
191
192 int
uiomove(void * cp,int n,struct uio * uio)193 uiomove(void *cp, int n, struct uio *uio)
194 {
195
196 return (uiomove_faultflag(cp, n, uio, 0));
197 }
198
199 int
uiomove_nofault(void * cp,int n,struct uio * uio)200 uiomove_nofault(void *cp, int n, struct uio *uio)
201 {
202
203 return (uiomove_faultflag(cp, n, uio, 1));
204 }
205
206 static int
uiomove_faultflag(void * cp,int n,struct uio * uio,int nofault)207 uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault)
208 {
209 struct iovec *iov;
210 size_t cnt;
211 int error, newflags, save;
212
213 save = error = 0;
214
215 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
216 ("uiomove: mode"));
217 KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
218 ("uiomove proc"));
219 KASSERT(uio->uio_resid >= 0,
220 ("%s: uio %p resid underflow", __func__, uio));
221
222 if (uio->uio_segflg == UIO_USERSPACE) {
223 newflags = TDP_DEADLKTREAT;
224 if (nofault) {
225 /*
226 * Fail if a non-spurious page fault occurs.
227 */
228 newflags |= TDP_NOFAULTING | TDP_RESETSPUR;
229 } else {
230 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
231 "Calling uiomove()");
232 }
233 save = curthread_pflags_set(newflags);
234 } else {
235 KASSERT(nofault == 0, ("uiomove: nofault"));
236 }
237
238 while (n > 0 && uio->uio_resid) {
239 KASSERT(uio->uio_iovcnt > 0,
240 ("%s: uio %p iovcnt underflow", __func__, uio));
241
242 iov = uio->uio_iov;
243 cnt = iov->iov_len;
244 if (cnt == 0) {
245 uio->uio_iov++;
246 uio->uio_iovcnt--;
247 continue;
248 }
249 if (cnt > n)
250 cnt = n;
251
252 switch (uio->uio_segflg) {
253 case UIO_USERSPACE:
254 maybe_yield();
255 if (uio->uio_rw == UIO_READ)
256 error = copyout(cp, iov->iov_base, cnt);
257 else
258 error = copyin(iov->iov_base, cp, cnt);
259 if (error)
260 goto out;
261 break;
262
263 case UIO_SYSSPACE:
264 if (uio->uio_rw == UIO_READ)
265 bcopy(cp, iov->iov_base, cnt);
266 else
267 bcopy(iov->iov_base, cp, cnt);
268 break;
269 case UIO_NOCOPY:
270 break;
271 }
272 iov->iov_base = (char *)iov->iov_base + cnt;
273 iov->iov_len -= cnt;
274 uio->uio_resid -= cnt;
275 uio->uio_offset += cnt;
276 cp = (char *)cp + cnt;
277 n -= cnt;
278 }
279 out:
280 if (save)
281 curthread_pflags_restore(save);
282 return (error);
283 }
284
285 /*
286 * Wrapper for uiomove() that validates the arguments against a known-good
287 * kernel buffer. Currently, uiomove accepts a signed (n) argument, which
288 * is almost definitely a bad thing, so we catch that here as well. We
289 * return a runtime failure, but it might be desirable to generate a runtime
290 * assertion failure instead.
291 */
292 int
uiomove_frombuf(void * buf,int buflen,struct uio * uio)293 uiomove_frombuf(void *buf, int buflen, struct uio *uio)
294 {
295 size_t offset, n;
296
297 if (uio->uio_offset < 0 || uio->uio_resid < 0 ||
298 (offset = uio->uio_offset) != uio->uio_offset)
299 return (EINVAL);
300 if (buflen <= 0 || offset >= buflen)
301 return (0);
302 if ((n = buflen - offset) > IOSIZE_MAX)
303 return (EINVAL);
304 return (uiomove((char *)buf + offset, n, uio));
305 }
306
307 /*
308 * Give next character to user as result of read.
309 */
310 int
ureadc(int c,struct uio * uio)311 ureadc(int c, struct uio *uio)
312 {
313 struct iovec *iov;
314 char *iov_base;
315
316 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
317 "Calling ureadc()");
318
319 again:
320 if (uio->uio_iovcnt == 0 || uio->uio_resid == 0)
321 panic("ureadc");
322 iov = uio->uio_iov;
323 if (iov->iov_len == 0) {
324 uio->uio_iovcnt--;
325 uio->uio_iov++;
326 goto again;
327 }
328 switch (uio->uio_segflg) {
329 case UIO_USERSPACE:
330 if (subyte(iov->iov_base, c) < 0)
331 return (EFAULT);
332 break;
333
334 case UIO_SYSSPACE:
335 iov_base = iov->iov_base;
336 *iov_base = c;
337 break;
338
339 case UIO_NOCOPY:
340 break;
341 }
342 iov->iov_base = (char *)iov->iov_base + 1;
343 iov->iov_len--;
344 uio->uio_resid--;
345 uio->uio_offset++;
346 return (0);
347 }
348
349 int
copyiniov(const struct iovec * iovp,u_int iovcnt,struct iovec ** iov,int error)350 copyiniov(const struct iovec *iovp, u_int iovcnt, struct iovec **iov, int error)
351 {
352 u_int iovlen;
353
354 *iov = NULL;
355 if (iovcnt > UIO_MAXIOV)
356 return (error);
357 iovlen = iovcnt * sizeof(struct iovec);
358 *iov = malloc(iovlen, M_IOV, M_WAITOK);
359 error = copyin(iovp, *iov, iovlen);
360 if (error) {
361 free(*iov, M_IOV);
362 *iov = NULL;
363 }
364 return (error);
365 }
366
367 int
copyinuio(const struct iovec * iovp,u_int iovcnt,struct uio ** uiop)368 copyinuio(const struct iovec *iovp, u_int iovcnt, struct uio **uiop)
369 {
370 struct iovec *iov;
371 struct uio *uio;
372 u_int iovlen;
373 int error, i;
374
375 *uiop = NULL;
376 if (iovcnt > UIO_MAXIOV)
377 return (EINVAL);
378 iovlen = iovcnt * sizeof(struct iovec);
379 uio = allocuio(iovcnt);
380 iov = uio->uio_iov;
381 error = copyin(iovp, iov, iovlen);
382 if (error != 0) {
383 freeuio(uio);
384 return (error);
385 }
386 uio->uio_iovcnt = iovcnt;
387 uio->uio_segflg = UIO_USERSPACE;
388 uio->uio_offset = -1;
389 uio->uio_resid = 0;
390 for (i = 0; i < iovcnt; i++) {
391 if (iov->iov_len > IOSIZE_MAX - uio->uio_resid) {
392 freeuio(uio);
393 return (EINVAL);
394 }
395 uio->uio_resid += iov->iov_len;
396 iov++;
397 }
398 *uiop = uio;
399 return (0);
400 }
401
402 struct uio *
allocuio(u_int iovcnt)403 allocuio(u_int iovcnt)
404 {
405 struct uio *uio;
406 int iovlen;
407
408 KASSERT(iovcnt <= UIO_MAXIOV,
409 ("Requested %u iovecs exceed UIO_MAXIOV", iovcnt));
410 iovlen = iovcnt * sizeof(struct iovec);
411 uio = malloc(iovlen + sizeof(*uio), M_IOV, M_WAITOK);
412 uio->uio_iov = (struct iovec *)(uio + 1);
413
414 return (uio);
415 }
416
417 void
freeuio(struct uio * uio)418 freeuio(struct uio *uio)
419 {
420 free(uio, M_IOV);
421 }
422
423 struct uio *
cloneuio(struct uio * uiop)424 cloneuio(struct uio *uiop)
425 {
426 struct iovec *iov;
427 struct uio *uio;
428 int iovlen;
429
430 iovlen = uiop->uio_iovcnt * sizeof(struct iovec);
431 uio = allocuio(uiop->uio_iovcnt);
432 iov = uio->uio_iov;
433 *uio = *uiop;
434 uio->uio_iov = iov;
435 bcopy(uiop->uio_iov, uio->uio_iov, iovlen);
436 return (uio);
437 }
438
439 /*
440 * Map some anonymous memory in user space of size sz, rounded up to the page
441 * boundary.
442 */
443 int
copyout_map(struct thread * td,vm_offset_t * addr,size_t sz)444 copyout_map(struct thread *td, vm_offset_t *addr, size_t sz)
445 {
446 struct vmspace *vms;
447 int error;
448 vm_size_t size;
449
450 vms = td->td_proc->p_vmspace;
451
452 /*
453 * Map somewhere after heap in process memory.
454 */
455 *addr = round_page((vm_offset_t)vms->vm_daddr +
456 lim_max(td, RLIMIT_DATA));
457
458 /* round size up to page boundary */
459 size = (vm_size_t)round_page(sz);
460 if (size == 0)
461 return (EINVAL);
462 error = vm_mmap_object(&vms->vm_map, addr, size, VM_PROT_READ |
463 VM_PROT_WRITE, VM_PROT_ALL, MAP_PRIVATE | MAP_ANON, NULL, 0,
464 FALSE, td);
465 return (error);
466 }
467
468 /*
469 * Unmap memory in user space.
470 */
471 int
copyout_unmap(struct thread * td,vm_offset_t addr,size_t sz)472 copyout_unmap(struct thread *td, vm_offset_t addr, size_t sz)
473 {
474 vm_map_t map;
475 vm_size_t size;
476
477 if (sz == 0)
478 return (0);
479
480 map = &td->td_proc->p_vmspace->vm_map;
481 size = (vm_size_t)round_page(sz);
482
483 if (vm_map_remove(map, addr, addr + size) != KERN_SUCCESS)
484 return (EINVAL);
485
486 return (0);
487 }
488
489 int32_t
fuword32(volatile const void * addr)490 fuword32(volatile const void *addr)
491 {
492 int rv;
493 int32_t val;
494
495 rv = fueword32(addr, &val);
496 return (rv == -1 ? -1 : val);
497 }
498
499 #ifdef _LP64
500 int64_t
fuword64(volatile const void * addr)501 fuword64(volatile const void *addr)
502 {
503 int rv;
504 int64_t val;
505
506 rv = fueword64(addr, &val);
507 return (rv == -1 ? -1 : val);
508 }
509 #endif /* _LP64 */
510
511 long
fuword(volatile const void * addr)512 fuword(volatile const void *addr)
513 {
514 long val;
515 int rv;
516
517 rv = fueword(addr, &val);
518 return (rv == -1 ? -1 : val);
519 }
520
521 uint32_t
casuword32(volatile uint32_t * addr,uint32_t old,uint32_t new)522 casuword32(volatile uint32_t *addr, uint32_t old, uint32_t new)
523 {
524 int rv;
525 uint32_t val;
526
527 rv = casueword32(addr, old, &val, new);
528 return (rv == -1 ? -1 : val);
529 }
530
531 u_long
casuword(volatile u_long * addr,u_long old,u_long new)532 casuword(volatile u_long *addr, u_long old, u_long new)
533 {
534 int rv;
535 u_long val;
536
537 rv = casueword(addr, old, &val, new);
538 return (rv == -1 ? -1 : val);
539 }
540