Lines Matching refs:bio
48 static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter) in bio_copy_from_iter() argument
53 bio_for_each_segment_all(bvec, bio, iter_all) { in bio_copy_from_iter()
79 static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter) in bio_copy_to_iter() argument
84 bio_for_each_segment_all(bvec, bio, iter_all) { in bio_copy_to_iter()
109 static int bio_uncopy_user(struct bio *bio) in bio_uncopy_user() argument
111 struct bio_map_data *bmd = bio->bi_private; in bio_uncopy_user()
122 else if (bio_data_dir(bio) == READ) in bio_uncopy_user()
123 ret = bio_copy_to_iter(bio, bmd->iter); in bio_uncopy_user()
125 bio_free_pages(bio); in bio_uncopy_user()
136 struct bio *bio; in bio_copy_user_iov() local
157 bio = bio_kmalloc(nr_pages, gfp_mask); in bio_copy_user_iov()
158 if (!bio) in bio_copy_user_iov()
160 bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, req_op(rq)); in bio_copy_user_iov()
192 if (bio_add_page(bio, page, bytes, offset) < bytes) { in bio_copy_user_iov()
203 map_data->offset += bio->bi_iter.bi_size; in bio_copy_user_iov()
210 ret = bio_copy_from_iter(bio, iter); in bio_copy_user_iov()
218 ret = bio_copy_from_iter(bio, &iter2); in bio_copy_user_iov()
223 zero_fill_bio(bio); in bio_copy_user_iov()
224 iov_iter_advance(iter, bio->bi_iter.bi_size); in bio_copy_user_iov()
227 bio->bi_private = bmd; in bio_copy_user_iov()
229 ret = blk_rq_append_bio(rq, bio); in bio_copy_user_iov()
235 bio_free_pages(bio); in bio_copy_user_iov()
236 bio_uninit(bio); in bio_copy_user_iov()
237 kfree(bio); in bio_copy_user_iov()
243 static void blk_mq_map_bio_put(struct bio *bio) in blk_mq_map_bio_put() argument
245 if (bio->bi_opf & REQ_ALLOC_CACHE) { in blk_mq_map_bio_put()
246 bio_put(bio); in blk_mq_map_bio_put()
248 bio_uninit(bio); in blk_mq_map_bio_put()
249 kfree(bio); in blk_mq_map_bio_put()
253 static struct bio *blk_rq_map_bio_alloc(struct request *rq, in blk_rq_map_bio_alloc()
256 struct bio *bio; in blk_rq_map_bio_alloc() local
259 bio = bio_alloc_bioset(NULL, nr_vecs, rq->cmd_flags, gfp_mask, in blk_rq_map_bio_alloc()
261 if (!bio) in blk_rq_map_bio_alloc()
264 bio = bio_kmalloc(nr_vecs, gfp_mask); in blk_rq_map_bio_alloc()
265 if (!bio) in blk_rq_map_bio_alloc()
267 bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq)); in blk_rq_map_bio_alloc()
269 return bio; in blk_rq_map_bio_alloc()
276 struct bio *bio; in bio_map_user_iov() local
282 bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask); in bio_map_user_iov()
283 if (!bio) in bio_map_user_iov()
285 ret = bio_iov_iter_get_pages(bio, iter); in bio_map_user_iov()
288 ret = blk_rq_append_bio(rq, bio); in bio_map_user_iov()
294 bio_release_pages(bio, false); in bio_map_user_iov()
296 blk_mq_map_bio_put(bio); in bio_map_user_iov()
300 static void bio_invalidate_vmalloc_pages(struct bio *bio) in bio_invalidate_vmalloc_pages() argument
303 if (bio->bi_private && !op_is_write(bio_op(bio))) { in bio_invalidate_vmalloc_pages()
306 for (i = 0; i < bio->bi_vcnt; i++) in bio_invalidate_vmalloc_pages()
307 len += bio->bi_io_vec[i].bv_len; in bio_invalidate_vmalloc_pages()
308 invalidate_kernel_vmap_range(bio->bi_private, len); in bio_invalidate_vmalloc_pages()
313 static void bio_map_kern_endio(struct bio *bio) in bio_map_kern_endio() argument
315 bio_invalidate_vmalloc_pages(bio); in bio_map_kern_endio()
316 bio_uninit(bio); in bio_map_kern_endio()
317 kfree(bio); in bio_map_kern_endio()
330 static struct bio *bio_map_kern(struct request_queue *q, void *data, in bio_map_kern()
340 struct bio *bio; in bio_map_kern() local
342 bio = bio_kmalloc(nr_pages, gfp_mask); in bio_map_kern()
343 if (!bio) in bio_map_kern()
345 bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); in bio_map_kern()
349 bio->bi_private = data; in bio_map_kern()
366 if (bio_add_page(bio, page, bytes, offset) < bytes) { in bio_map_kern()
368 bio_uninit(bio); in bio_map_kern()
369 kfree(bio); in bio_map_kern()
378 bio->bi_end_io = bio_map_kern_endio; in bio_map_kern()
379 return bio; in bio_map_kern()
382 static void bio_copy_kern_endio(struct bio *bio) in bio_copy_kern_endio() argument
384 bio_free_pages(bio); in bio_copy_kern_endio()
385 bio_uninit(bio); in bio_copy_kern_endio()
386 kfree(bio); in bio_copy_kern_endio()
389 static void bio_copy_kern_endio_read(struct bio *bio) in bio_copy_kern_endio_read() argument
391 char *p = bio->bi_private; in bio_copy_kern_endio_read()
395 bio_for_each_segment_all(bvec, bio, iter_all) { in bio_copy_kern_endio_read()
400 bio_copy_kern_endio(bio); in bio_copy_kern_endio_read()
414 static struct bio *bio_copy_kern(struct request_queue *q, void *data, in bio_copy_kern()
420 struct bio *bio; in bio_copy_kern() local
431 bio = bio_kmalloc(nr_pages, gfp_mask); in bio_copy_kern()
432 if (!bio) in bio_copy_kern()
434 bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); in bio_copy_kern()
450 if (bio_add_page(bio, page, bytes, 0) < bytes) in bio_copy_kern()
458 bio->bi_end_io = bio_copy_kern_endio_read; in bio_copy_kern()
459 bio->bi_private = data; in bio_copy_kern()
461 bio->bi_end_io = bio_copy_kern_endio; in bio_copy_kern()
464 return bio; in bio_copy_kern()
467 bio_free_pages(bio); in bio_copy_kern()
468 bio_uninit(bio); in bio_copy_kern()
469 kfree(bio); in bio_copy_kern()
477 int blk_rq_append_bio(struct request *rq, struct bio *bio) in blk_rq_append_bio() argument
485 ret = bio_split_rw_at(bio, lim, &nr_segs, max_bytes); in blk_rq_append_bio()
493 if (rq->bio) { in blk_rq_append_bio()
494 if (!ll_back_merge_fn(rq, bio, nr_segs)) in blk_rq_append_bio()
496 rq->biotail->bi_next = bio; in blk_rq_append_bio()
497 rq->biotail = bio; in blk_rq_append_bio()
498 rq->__data_len += bio->bi_iter.bi_size; in blk_rq_append_bio()
499 bio_crypt_free_ctx(bio); in blk_rq_append_bio()
504 rq->bio = rq->biotail = bio; in blk_rq_append_bio()
505 rq->__data_len = bio->bi_iter.bi_size; in blk_rq_append_bio()
514 struct bio *bio; in blk_rq_map_user_bvec() local
521 bio = blk_rq_map_bio_alloc(rq, 0, GFP_KERNEL); in blk_rq_map_user_bvec()
522 if (!bio) in blk_rq_map_user_bvec()
524 bio_iov_bvec_set(bio, iter); in blk_rq_map_user_bvec()
526 ret = blk_rq_append_bio(rq, bio); in blk_rq_map_user_bvec()
528 blk_mq_map_bio_put(bio); in blk_rq_map_user_bvec()
553 struct bio *bio = NULL; in blk_rq_map_user_iov() local
591 if (!bio) in blk_rq_map_user_iov()
592 bio = rq->bio; in blk_rq_map_user_iov()
598 blk_rq_unmap_user(bio); in blk_rq_map_user_iov()
600 rq->bio = NULL; in blk_rq_map_user_iov()
664 int blk_rq_unmap_user(struct bio *bio) in blk_rq_unmap_user() argument
666 struct bio *next_bio; in blk_rq_unmap_user()
669 while (bio) { in blk_rq_unmap_user()
670 if (bio->bi_private) { in blk_rq_unmap_user()
671 ret2 = bio_uncopy_user(bio); in blk_rq_unmap_user()
675 bio_release_pages(bio, bio_data_dir(bio) == READ); in blk_rq_unmap_user()
678 if (bio_integrity(bio)) in blk_rq_unmap_user()
679 bio_integrity_unmap_user(bio); in blk_rq_unmap_user()
681 next_bio = bio; in blk_rq_unmap_user()
682 bio = bio->bi_next; in blk_rq_unmap_user()
708 struct bio *bio; in blk_rq_map_kern() local
718 bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); in blk_rq_map_kern()
720 bio = bio_map_kern(q, kbuf, len, gfp_mask); in blk_rq_map_kern()
722 if (IS_ERR(bio)) in blk_rq_map_kern()
723 return PTR_ERR(bio); in blk_rq_map_kern()
725 bio->bi_opf &= ~REQ_OP_MASK; in blk_rq_map_kern()
726 bio->bi_opf |= req_op(rq); in blk_rq_map_kern()
728 ret = blk_rq_append_bio(rq, bio); in blk_rq_map_kern()
730 bio_uninit(bio); in blk_rq_map_kern()
731 kfree(bio); in blk_rq_map_kern()