23 #define bio_iter_iovec(bio, iter)				\  argument
24 	bvec_iter_bvec((bio)->bi_io_vec, (iter))
26 #define bio_iter_page(bio, iter)				\  argument
27 	bvec_iter_page((bio)->bi_io_vec, (iter))
28 #define bio_iter_len(bio, iter)					\  argument
29 	bvec_iter_len((bio)->bi_io_vec, (iter))
30 #define bio_iter_offset(bio, iter)				\  argument
31 	bvec_iter_offset((bio)->bi_io_vec, (iter))
33 #define bio_page(bio)		bio_iter_page((bio), (bio)->bi_iter)  argument
34 #define bio_offset(bio)		bio_iter_offset((bio), (bio)->bi_iter)  argument
35 #define bio_iovec(bio)		bio_iter_iovec((bio), (bio)->bi_iter)  argument
40 #define bio_sectors(bio)	bvec_iter_sectors((bio)->bi_iter)  argument
41 #define bio_end_sector(bio)	bvec_iter_end_sector((bio)->bi_iter)  argument
46 #define bio_data_dir(bio) \  argument
47 	(op_is_write(bio_op(bio)) ? WRITE : READ)
52 static inline bool bio_has_data(struct bio *bio)  in bio_has_data()  argument
54 	if (bio &&  in bio_has_data()
55 	    bio->bi_iter.bi_size &&  in bio_has_data()
56 	    bio_op(bio) != REQ_OP_DISCARD &&  in bio_has_data()
57 	    bio_op(bio) != REQ_OP_SECURE_ERASE &&  in bio_has_data()
58 	    bio_op(bio) != REQ_OP_WRITE_ZEROES)  in bio_has_data()
64 static inline bool bio_no_advance_iter(const struct bio *bio)  in bio_no_advance_iter()  argument
66 	return bio_op(bio) == REQ_OP_DISCARD ||  in bio_no_advance_iter()
67 	       bio_op(bio) == REQ_OP_SECURE_ERASE ||  in bio_no_advance_iter()
68 	       bio_op(bio) == REQ_OP_WRITE_ZEROES;  in bio_no_advance_iter()
71 static inline void *bio_data(struct bio *bio)  in bio_data()  argument
73 	if (bio_has_data(bio))  in bio_data()
74 		return page_address(bio_page(bio)) + bio_offset(bio);  in bio_data()
79 static inline bool bio_next_segment(const struct bio *bio,  in bio_next_segment()  argument
82 	if (iter->idx >= bio->bi_vcnt)  in bio_next_segment()
85 	bvec_advance(&bio->bi_io_vec[iter->idx], iter);  in bio_next_segment()
93 #define bio_for_each_segment_all(bvl, bio, iter) \  argument
94 	for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); )
96 static inline void bio_advance_iter(const struct bio *bio,  in bio_advance_iter()  argument
101 	if (bio_no_advance_iter(bio))  in bio_advance_iter()
104 		bvec_iter_advance(bio->bi_io_vec, iter, bytes);  in bio_advance_iter()
109 static inline void bio_advance_iter_single(const struct bio *bio,  in bio_advance_iter_single()  argument
115 	if (bio_no_advance_iter(bio))  in bio_advance_iter_single()
118 		bvec_iter_advance_single(bio->bi_io_vec, iter, bytes);  in bio_advance_iter_single()
121 void __bio_advance(struct bio *, unsigned bytes);
134 static inline void bio_advance(struct bio *bio, unsigned int nbytes)  in bio_advance()  argument
136 	if (nbytes == bio->bi_iter.bi_size) {  in bio_advance()
137 		bio->bi_iter.bi_size = 0;  in bio_advance()
140 	__bio_advance(bio, nbytes);  in bio_advance()
143 #define __bio_for_each_segment(bvl, bio, iter, start)			\  argument
146 		((bvl = bio_iter_iovec((bio), (iter))), 1);		\
147 	     bio_advance_iter_single((bio), &(iter), (bvl).bv_len))
149 #define bio_for_each_segment(bvl, bio, iter)				\  argument
150 	__bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
152 #define __bio_for_each_bvec(bvl, bio, iter, start)		\  argument
155 		((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \
156 	     bio_advance_iter_single((bio), &(iter), (bvl).bv_len))
159 #define bio_for_each_bvec(bvl, bio, iter)			\  argument
160 	__bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)
166 #define bio_for_each_bvec_all(bvl, bio, i)		\  argument
167 	for (i = 0, bvl = bio_first_bvec_all(bio);	\
168 	     i < (bio)->bi_vcnt; i++, bvl++)
172 static inline unsigned bio_segments(struct bio *bio)  in bio_segments()  argument
183 	switch (bio_op(bio)) {  in bio_segments()
192 	bio_for_each_segment(bv, bio, iter)  in bio_segments()
212 static inline void bio_get(struct bio *bio)  in bio_get()  argument
214 	bio->bi_flags |= (1 << BIO_REFFED);  in bio_get()
216 	atomic_inc(&bio->__bi_cnt);  in bio_get()
219 static inline void bio_cnt_set(struct bio *bio, unsigned int count)  in bio_cnt_set()  argument
222 		bio->bi_flags |= (1 << BIO_REFFED);  in bio_cnt_set()
225 	atomic_set(&bio->__bi_cnt, count);  in bio_cnt_set()
228 static inline bool bio_flagged(struct bio *bio, unsigned int bit)  in bio_flagged()  argument
230 	return bio->bi_flags & (1U << bit);  in bio_flagged()
233 static inline void bio_set_flag(struct bio *bio, unsigned int bit)  in bio_set_flag()  argument
235 	bio->bi_flags |= (1U << bit);  in bio_set_flag()
238 static inline void bio_clear_flag(struct bio *bio, unsigned int bit)  in bio_clear_flag()  argument
240 	bio->bi_flags &= ~(1U << bit);  in bio_clear_flag()
243 static inline struct bio_vec *bio_first_bvec_all(struct bio *bio)  in bio_first_bvec_all()  argument
245 	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));  in bio_first_bvec_all()
246 	return bio->bi_io_vec;  in bio_first_bvec_all()
249 static inline struct page *bio_first_page_all(struct bio *bio)  in bio_first_page_all()  argument
251 	return bio_first_bvec_all(bio)->bv_page;  in bio_first_page_all()
254 static inline struct folio *bio_first_folio_all(struct bio *bio)  in bio_first_folio_all()  argument
256 	return page_folio(bio_first_page_all(bio));  in bio_first_folio_all()
259 static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)  in bio_last_bvec_all()  argument
261 	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));  in bio_last_bvec_all()
262 	return &bio->bi_io_vec[bio->bi_vcnt - 1];  in bio_last_bvec_all()
282 static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio,  in bio_first_folio()  argument
285 	struct bio_vec *bvec = bio_first_bvec_all(bio) + i;  in bio_first_folio()
287 	if (unlikely(i >= bio->bi_vcnt)) {  in bio_first_folio()
301 static inline void bio_next_folio(struct folio_iter *fi, struct bio *bio)  in bio_next_folio()  argument
310 		bio_first_folio(fi, bio, fi->_i + 1);  in bio_next_folio()
319 #define bio_for_each_folio_all(fi, bio)				\  argument
320 	for (bio_first_folio(&fi, bio, 0); fi.folio; bio_next_folio(&fi, bio))
322 void bio_trim(struct bio *bio, sector_t offset, sector_t size);
323 extern struct bio *bio_split(struct bio *bio, int sectors,
325 int bio_split_rw_at(struct bio *bio, const struct queue_limits *lim,
338 static inline struct bio *bio_next_split(struct bio *bio, int sectors,  in bio_next_split()  argument
341 	if (sectors >= bio_sectors(bio))  in bio_next_split()
342 		return bio;  in bio_next_split()
344 	return bio_split(bio, sectors, gfp, bs);  in bio_next_split()
356 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
359 struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask);
360 extern void bio_put(struct bio *);
362 struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src,
364 int bio_init_clone(struct block_device *bdev, struct bio *bio,
365 		struct bio *bio_src, gfp_t gfp);
369 static inline struct bio *bio_alloc(struct block_device *bdev,  in bio_alloc()
375 void submit_bio(struct bio *bio);
377 extern void bio_endio(struct bio *);
379 static inline void bio_io_error(struct bio *bio)  in bio_io_error()  argument
381 	bio->bi_status = BLK_STS_IOERR;  in bio_io_error()
382 	bio_endio(bio);  in bio_io_error()
385 static inline void bio_wouldblock_error(struct bio *bio)  in bio_wouldblock_error()  argument
387 	bio_set_flag(bio, BIO_QUIET);  in bio_wouldblock_error()
388 	bio->bi_status = BLK_STS_AGAIN;  in bio_wouldblock_error()
389 	bio_endio(bio);  in bio_wouldblock_error()
406 extern int submit_bio_wait(struct bio *bio);
407 void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
409 extern void bio_uninit(struct bio *);
410 void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf);
411 void bio_chain(struct bio *, struct bio *);
413 int __must_check bio_add_page(struct bio *bio, struct page *page, unsigned len,
415 bool __must_check bio_add_folio(struct bio *bio, struct folio *folio,
417 void __bio_add_page(struct bio *bio, struct page *page,
419 void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len,
421 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
422 void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter);
423 void __bio_release_pages(struct bio *bio, bool mark_dirty);
424 extern void bio_set_pages_dirty(struct bio *bio);
425 extern void bio_check_pages_dirty(struct bio *bio);
427 extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
428 			       struct bio *src, struct bvec_iter *src_iter);
429 extern void bio_copy_data(struct bio *dst, struct bio *src);
430 extern void bio_free_pages(struct bio *bio);
431 void guard_bio_eod(struct bio *bio);
432 void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter);
434 static inline void zero_fill_bio(struct bio *bio)  in zero_fill_bio()  argument
436 	zero_fill_bio_iter(bio, bio->bi_iter);  in zero_fill_bio()
439 static inline void bio_release_pages(struct bio *bio, bool mark_dirty)  in bio_release_pages()  argument
441 	if (bio_flagged(bio, BIO_PAGE_PINNED))  in bio_release_pages()
442 		__bio_release_pages(bio, mark_dirty);  in bio_release_pages()
445 #define bio_dev(bio) \  argument
446 	disk_devt((bio)->bi_bdev->bd_disk)
449 void bio_associate_blkg(struct bio *bio);
450 void bio_associate_blkg_from_css(struct bio *bio,
452 void bio_clone_blkg_association(struct bio *dst, struct bio *src);
453 void blkcg_punt_bio_submit(struct bio *bio);
455 static inline void bio_associate_blkg(struct bio *bio) { }  in bio_associate_blkg()  argument
456 static inline void bio_associate_blkg_from_css(struct bio *bio,  in bio_associate_blkg_from_css()  argument
459 static inline void bio_clone_blkg_association(struct bio *dst,  in bio_clone_blkg_association()
460 					      struct bio *src) { }  in bio_clone_blkg_association()
461 static inline void blkcg_punt_bio_submit(struct bio *bio)  in blkcg_punt_bio_submit()  argument
463 	submit_bio(bio);  in blkcg_punt_bio_submit()
467 static inline void bio_set_dev(struct bio *bio, struct block_device *bdev)  in bio_set_dev()  argument
469 	bio_clear_flag(bio, BIO_REMAPPED);  in bio_set_dev()
470 	if (bio->bi_bdev != bdev)  in bio_set_dev()
471 		bio_clear_flag(bio, BIO_BPS_THROTTLED);  in bio_set_dev()
472 	bio->bi_bdev = bdev;  in bio_set_dev()
473 	bio_associate_blkg(bio);  in bio_set_dev()
484 	struct bio *head;
485 	struct bio *tail;
500 #define bio_list_for_each(bio, bl) \  argument
501 	for (bio = (bl)->head; bio; bio = bio->bi_next)
506 	struct bio *bio;  in bio_list_size()  local
508 	bio_list_for_each(bio, bl)  in bio_list_size()
514 static inline void bio_list_add(struct bio_list *bl, struct bio *bio)  in bio_list_add()  argument
516 	bio->bi_next = NULL;  in bio_list_add()
519 		bl->tail->bi_next = bio;  in bio_list_add()
521 		bl->head = bio;  in bio_list_add()
523 	bl->tail = bio;  in bio_list_add()
526 static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)  in bio_list_add_head()  argument
528 	bio->bi_next = bl->head;  in bio_list_add_head()
530 	bl->head = bio;  in bio_list_add_head()
533 		bl->tail = bio;  in bio_list_add_head()
570 static inline struct bio *bio_list_peek(struct bio_list *bl)  in bio_list_peek()
575 static inline struct bio *bio_list_pop(struct bio_list *bl)  in bio_list_pop()
577 	struct bio *bio = bl->head;  in bio_list_pop()  local
579 	if (bio) {  in bio_list_pop()
584 		bio->bi_next = NULL;  in bio_list_pop()
587 	return bio;  in bio_list_pop()
590 static inline struct bio *bio_list_get(struct bio_list *bl)  in bio_list_get()
592 	struct bio *bio = bl->head;  in bio_list_get()  local
596 	return bio;  in bio_list_get()
603 static inline void bio_inc_remaining(struct bio *bio)  in bio_inc_remaining()  argument
605 	bio_set_flag(bio, BIO_CHAIN);  in bio_inc_remaining()
607 	atomic_inc(&bio->__bi_remaining);  in bio_inc_remaining()
658 static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb)  in bio_set_polled()  argument
660 	bio->bi_opf |= REQ_POLLED;  in bio_set_polled()
662 		bio->bi_opf |= REQ_NOWAIT;  in bio_set_polled()
665 static inline void bio_clear_polled(struct bio *bio)  in bio_clear_polled()  argument
667 	bio->bi_opf &= ~REQ_POLLED;  in bio_clear_polled()
679 static inline bool bio_is_zone_append(struct bio *bio)  in bio_is_zone_append()  argument
683 	return bio_op(bio) == REQ_OP_ZONE_APPEND ||  in bio_is_zone_append()
684 		bio_flagged(bio, BIO_EMULATES_ZONE_APPEND);  in bio_is_zone_append()
687 struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
689 struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new);
691 struct bio *blk_alloc_discard_bio(struct block_device *bdev,