1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
27 /* All Rights Reserved */
28
29 /*
30 * University Copyright- Copyright (c) 1982, 1986, 1988
31 * The Regents of the University of California
32 * All Rights Reserved
33 *
34 * University Acknowledgment- Portions of this document are derived from
35 * software developed by the University of California, Berkeley, and its
36 * contributors.
37 */
38 /*
39 * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
40 */
41
42 #ifdef _KERNEL
43
44 #include <sys/types.h>
45 #include <sys/uio_impl.h>
46 #include <sys/sysmacros.h>
47 #include <sys/string.h>
48 #include <linux/kmap_compat.h>
49 #include <linux/uaccess.h>
50
51 /*
52 * Move "n" bytes at byte address "p"; "rw" indicates the direction
53 * of the move, and the I/O parameters are provided in "uio", which is
54 * update to reflect the data which was moved. Returns 0 on success or
55 * a non-zero errno on failure.
56 */
57 static int
zfs_uiomove_iov(void * p,size_t n,zfs_uio_rw_t rw,zfs_uio_t * uio)58 zfs_uiomove_iov(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
59 {
60 const struct iovec *iov = uio->uio_iov;
61 size_t skip = uio->uio_skip;
62 ulong_t cnt;
63
64 while (n && uio->uio_resid) {
65 cnt = MIN(iov->iov_len - skip, n);
66 switch (uio->uio_segflg) {
67 case UIO_USERSPACE:
68 /*
69 * p = kernel data pointer
70 * iov->iov_base = user data pointer
71 */
72 if (rw == UIO_READ) {
73 if (copy_to_user(iov->iov_base+skip, p, cnt))
74 return (EFAULT);
75 } else {
76 unsigned long b_left = 0;
77 if (uio->uio_fault_disable) {
78 if (!zfs_access_ok(VERIFY_READ,
79 (iov->iov_base + skip), cnt)) {
80 return (EFAULT);
81 }
82 pagefault_disable();
83 b_left =
84 __copy_from_user_inatomic(p,
85 (iov->iov_base + skip), cnt);
86 pagefault_enable();
87 } else {
88 b_left =
89 copy_from_user(p,
90 (iov->iov_base + skip), cnt);
91 }
92 if (b_left > 0) {
93 unsigned long c_bytes =
94 cnt - b_left;
95 uio->uio_skip += c_bytes;
96 ASSERT3U(uio->uio_skip, <,
97 iov->iov_len);
98 uio->uio_resid -= c_bytes;
99 uio->uio_loffset += c_bytes;
100 return (EFAULT);
101 }
102 }
103 break;
104 case UIO_SYSSPACE:
105 if (rw == UIO_READ)
106 memcpy(iov->iov_base + skip, p, cnt);
107 else
108 memcpy(p, iov->iov_base + skip, cnt);
109 break;
110 default:
111 ASSERT(0);
112 }
113 skip += cnt;
114 if (skip == iov->iov_len) {
115 skip = 0;
116 uio->uio_iov = (++iov);
117 uio->uio_iovcnt--;
118 }
119 uio->uio_skip = skip;
120 uio->uio_resid -= cnt;
121 uio->uio_loffset += cnt;
122 p = (caddr_t)p + cnt;
123 n -= cnt;
124 }
125 return (0);
126 }
127
128 static int
zfs_uiomove_bvec_impl(void * p,size_t n,zfs_uio_rw_t rw,zfs_uio_t * uio)129 zfs_uiomove_bvec_impl(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
130 {
131 const struct bio_vec *bv = uio->uio_bvec;
132 size_t skip = uio->uio_skip;
133 ulong_t cnt;
134
135 while (n && uio->uio_resid) {
136 void *paddr;
137 cnt = MIN(bv->bv_len - skip, n);
138
139 paddr = zfs_kmap_atomic(bv->bv_page);
140 if (rw == UIO_READ) {
141 /* Copy from buffer 'p' to the bvec data */
142 memcpy(paddr + bv->bv_offset + skip, p, cnt);
143 } else {
144 /* Copy from bvec data to buffer 'p' */
145 memcpy(p, paddr + bv->bv_offset + skip, cnt);
146 }
147 zfs_kunmap_atomic(paddr);
148
149 skip += cnt;
150 if (skip == bv->bv_len) {
151 skip = 0;
152 uio->uio_bvec = (++bv);
153 uio->uio_iovcnt--;
154 }
155 uio->uio_skip = skip;
156 uio->uio_resid -= cnt;
157 uio->uio_loffset += cnt;
158 p = (caddr_t)p + cnt;
159 n -= cnt;
160 }
161 return (0);
162 }
163
164 #ifdef HAVE_BLK_MQ
165 static void
zfs_copy_bvec(void * p,size_t skip,size_t cnt,zfs_uio_rw_t rw,struct bio_vec * bv)166 zfs_copy_bvec(void *p, size_t skip, size_t cnt, zfs_uio_rw_t rw,
167 struct bio_vec *bv)
168 {
169 void *paddr;
170
171 paddr = zfs_kmap_atomic(bv->bv_page);
172 if (rw == UIO_READ) {
173 /* Copy from buffer 'p' to the bvec data */
174 memcpy(paddr + bv->bv_offset + skip, p, cnt);
175 } else {
176 /* Copy from bvec data to buffer 'p' */
177 memcpy(p, paddr + bv->bv_offset + skip, cnt);
178 }
179 zfs_kunmap_atomic(paddr);
180 }
181
182 /*
183 * Copy 'n' bytes of data between the buffer p[] and the data represented
184 * by the request in the uio.
185 */
186 static int
zfs_uiomove_bvec_rq(void * p,size_t n,zfs_uio_rw_t rw,zfs_uio_t * uio)187 zfs_uiomove_bvec_rq(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
188 {
189 struct request *rq = uio->rq;
190 struct bio_vec bv;
191 struct req_iterator iter;
192 size_t this_seg_start; /* logical offset */
193 size_t this_seg_end; /* logical offset */
194 size_t skip_in_seg;
195 size_t copy_from_seg;
196 size_t orig_loffset;
197 int copied = 0;
198
199 /*
200 * Get the original logical offset of this entire request (because
201 * uio->uio_loffset will be modified over time).
202 */
203 orig_loffset = io_offset(NULL, rq);
204 this_seg_start = orig_loffset;
205
206 rq_for_each_segment(bv, rq, iter) {
207 /*
208 * Lookup what the logical offset of the last byte of this
209 * segment is.
210 */
211 this_seg_end = this_seg_start + bv.bv_len - 1;
212
213 /*
214 * We only need to operate on segments that have data we're
215 * copying.
216 */
217 if (uio->uio_loffset >= this_seg_start &&
218 uio->uio_loffset <= this_seg_end) {
219 /*
220 * Some, or all, of the data in this segment needs to be
221 * copied.
222 */
223
224 /*
225 * We may be not be copying from the first byte in the
226 * segment. Figure out how many bytes to skip copying
227 * from the beginning of this segment.
228 */
229 skip_in_seg = uio->uio_loffset - this_seg_start;
230
231 /*
232 * Calculate the total number of bytes from this
233 * segment that we will be copying.
234 */
235 copy_from_seg = MIN(bv.bv_len - skip_in_seg, n);
236
237 /* Copy the bytes */
238 zfs_copy_bvec(p, skip_in_seg, copy_from_seg, rw, &bv);
239 p = ((char *)p) + copy_from_seg;
240
241 n -= copy_from_seg;
242 uio->uio_resid -= copy_from_seg;
243 uio->uio_loffset += copy_from_seg;
244 copied = 1; /* We copied some data */
245 }
246
247 this_seg_start = this_seg_end + 1;
248 }
249
250 if (!copied) {
251 /* Didn't copy anything */
252 uio->uio_resid = 0;
253 }
254 return (0);
255 }
256 #endif
257
258 static int
zfs_uiomove_bvec(void * p,size_t n,zfs_uio_rw_t rw,zfs_uio_t * uio)259 zfs_uiomove_bvec(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
260 {
261 #ifdef HAVE_BLK_MQ
262 if (uio->rq != NULL)
263 return (zfs_uiomove_bvec_rq(p, n, rw, uio));
264 #else
265 ASSERT3P(uio->rq, ==, NULL);
266 #endif
267 return (zfs_uiomove_bvec_impl(p, n, rw, uio));
268 }
269
270 #if defined(HAVE_VFS_IOV_ITER)
271 static int
zfs_uiomove_iter(void * p,size_t n,zfs_uio_rw_t rw,zfs_uio_t * uio,boolean_t revert)272 zfs_uiomove_iter(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio,
273 boolean_t revert)
274 {
275 size_t cnt = MIN(n, uio->uio_resid);
276
277 if (uio->uio_skip)
278 iov_iter_advance(uio->uio_iter, uio->uio_skip);
279
280 if (rw == UIO_READ)
281 cnt = copy_to_iter(p, cnt, uio->uio_iter);
282 else
283 cnt = copy_from_iter(p, cnt, uio->uio_iter);
284
285 /*
286 * When operating on a full pipe no bytes are processed.
287 * In which case return EFAULT which is converted to EAGAIN
288 * by the kernel's generic_file_splice_read() function.
289 */
290 if (cnt == 0)
291 return (EFAULT);
292
293 /*
294 * Revert advancing the uio_iter. This is set by zfs_uiocopy()
295 * to avoid consuming the uio and its iov_iter structure.
296 */
297 if (revert)
298 iov_iter_revert(uio->uio_iter, cnt);
299
300 uio->uio_resid -= cnt;
301 uio->uio_loffset += cnt;
302
303 return (0);
304 }
305 #endif
306
307 int
zfs_uiomove(void * p,size_t n,zfs_uio_rw_t rw,zfs_uio_t * uio)308 zfs_uiomove(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
309 {
310 if (uio->uio_segflg == UIO_BVEC)
311 return (zfs_uiomove_bvec(p, n, rw, uio));
312 #if defined(HAVE_VFS_IOV_ITER)
313 else if (uio->uio_segflg == UIO_ITER)
314 return (zfs_uiomove_iter(p, n, rw, uio, B_FALSE));
315 #endif
316 else
317 return (zfs_uiomove_iov(p, n, rw, uio));
318 }
319 EXPORT_SYMBOL(zfs_uiomove);
320
321 /*
322 * Fault in the pages of the first n bytes specified by the uio structure.
323 * 1 byte in each page is touched and the uio struct is unmodified. Any
324 * error will terminate the process as this is only a best attempt to get
325 * the pages resident.
326 */
327 int
zfs_uio_prefaultpages(ssize_t n,zfs_uio_t * uio)328 zfs_uio_prefaultpages(ssize_t n, zfs_uio_t *uio)
329 {
330 if (uio->uio_segflg == UIO_SYSSPACE || uio->uio_segflg == UIO_BVEC) {
331 /* There's never a need to fault in kernel pages */
332 return (0);
333 #if defined(HAVE_VFS_IOV_ITER)
334 } else if (uio->uio_segflg == UIO_ITER) {
335 /*
336 * At least a Linux 4.9 kernel, iov_iter_fault_in_readable()
337 * can be relied on to fault in user pages when referenced.
338 */
339 if (iov_iter_fault_in_readable(uio->uio_iter, n))
340 return (EFAULT);
341 #endif
342 } else {
343 /* Fault in all user pages */
344 ASSERT3S(uio->uio_segflg, ==, UIO_USERSPACE);
345 const struct iovec *iov = uio->uio_iov;
346 int iovcnt = uio->uio_iovcnt;
347 size_t skip = uio->uio_skip;
348 uint8_t tmp;
349 caddr_t p;
350
351 for (; n > 0 && iovcnt > 0; iov++, iovcnt--, skip = 0) {
352 ulong_t cnt = MIN(iov->iov_len - skip, n);
353 /* empty iov */
354 if (cnt == 0)
355 continue;
356 n -= cnt;
357 /* touch each page in this segment. */
358 p = iov->iov_base + skip;
359 while (cnt) {
360 if (copy_from_user(&tmp, p, 1))
361 return (EFAULT);
362 ulong_t incr = MIN(cnt, PAGESIZE);
363 p += incr;
364 cnt -= incr;
365 }
366 /* touch the last byte in case it straddles a page. */
367 p--;
368 if (copy_from_user(&tmp, p, 1))
369 return (EFAULT);
370 }
371 }
372
373 return (0);
374 }
375 EXPORT_SYMBOL(zfs_uio_prefaultpages);
376
377 /*
378 * The same as zfs_uiomove() but doesn't modify uio structure.
379 * return in cbytes how many bytes were copied.
380 */
381 int
zfs_uiocopy(void * p,size_t n,zfs_uio_rw_t rw,zfs_uio_t * uio,size_t * cbytes)382 zfs_uiocopy(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio, size_t *cbytes)
383 {
384 zfs_uio_t uio_copy;
385 int ret;
386
387 memcpy(&uio_copy, uio, sizeof (zfs_uio_t));
388
389 if (uio->uio_segflg == UIO_BVEC)
390 ret = zfs_uiomove_bvec(p, n, rw, &uio_copy);
391 #if defined(HAVE_VFS_IOV_ITER)
392 else if (uio->uio_segflg == UIO_ITER)
393 ret = zfs_uiomove_iter(p, n, rw, &uio_copy, B_TRUE);
394 #endif
395 else
396 ret = zfs_uiomove_iov(p, n, rw, &uio_copy);
397
398 *cbytes = uio->uio_resid - uio_copy.uio_resid;
399
400 return (ret);
401 }
402 EXPORT_SYMBOL(zfs_uiocopy);
403
404 /*
405 * Drop the next n chars out of *uio.
406 */
407 void
zfs_uioskip(zfs_uio_t * uio,size_t n)408 zfs_uioskip(zfs_uio_t *uio, size_t n)
409 {
410 if (n > uio->uio_resid)
411 return;
412 /*
413 * When using a uio with a struct request, we simply
414 * use uio_loffset as a pointer to the next logical byte to
415 * copy in the request. We don't have to do any fancy
416 * accounting with uio_bvec/uio_iovcnt since we don't use
417 * them.
418 */
419 if (uio->uio_segflg == UIO_BVEC && uio->rq == NULL) {
420 uio->uio_skip += n;
421 while (uio->uio_iovcnt &&
422 uio->uio_skip >= uio->uio_bvec->bv_len) {
423 uio->uio_skip -= uio->uio_bvec->bv_len;
424 uio->uio_bvec++;
425 uio->uio_iovcnt--;
426 }
427 #if defined(HAVE_VFS_IOV_ITER)
428 } else if (uio->uio_segflg == UIO_ITER) {
429 iov_iter_advance(uio->uio_iter, n);
430 #endif
431 } else {
432 uio->uio_skip += n;
433 while (uio->uio_iovcnt &&
434 uio->uio_skip >= uio->uio_iov->iov_len) {
435 uio->uio_skip -= uio->uio_iov->iov_len;
436 uio->uio_iov++;
437 uio->uio_iovcnt--;
438 }
439 }
440 uio->uio_loffset += n;
441 uio->uio_resid -= n;
442 }
443 EXPORT_SYMBOL(zfs_uioskip);
444
445 #endif /* _KERNEL */
446