215 static inline bool blk_rq_is_passthrough(struct request *rq)  in blk_rq_is_passthrough()  argument
217 	return blk_op_is_passthrough(rq->cmd_flags);  in blk_rq_is_passthrough()
227 #define rq_data_dir(rq)		(op_is_write(req_op(rq)) ? WRITE : READ)  argument
229 #define rq_dma_dir(rq) \  argument
230 	(op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)
243 static inline void rq_list_add_tail(struct rq_list *rl, struct request *rq)  in rq_list_add_tail()  argument
245 	rq->rq_next = NULL;  in rq_list_add_tail()
247 		rl->tail->rq_next = rq;  in rq_list_add_tail()
249 		rl->head = rq;  in rq_list_add_tail()
250 	rl->tail = rq;  in rq_list_add_tail()
253 static inline void rq_list_add_head(struct rq_list *rl, struct request *rq)  in rq_list_add_head()  argument
255 	rq->rq_next = rl->head;  in rq_list_add_head()
256 	rl->head = rq;  in rq_list_add_head()
258 		rl->tail = rq;  in rq_list_add_head()
263 	struct request *rq = rl->head;  in rq_list_pop()  local
265 	if (rq) {  in rq_list_pop()
269 		rq->rq_next = NULL;  in rq_list_pop()
272 	return rq;  in rq_list_pop()
539 	struct request *rq;  member
658 	void (*show_rq)(struct seq_file *m, struct request *rq);
723 void blk_mq_free_request(struct request *rq);
724 int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,
782 u32 blk_mq_unique_tag(struct request *rq);
798 static inline enum mq_rq_state blk_mq_rq_state(struct request *rq)  in blk_mq_rq_state()  argument
800 	return READ_ONCE(rq->state);  in blk_mq_rq_state()
803 static inline int blk_mq_request_started(struct request *rq)  in blk_mq_request_started()  argument
805 	return blk_mq_rq_state(rq) != MQ_RQ_IDLE;  in blk_mq_request_started()
808 static inline int blk_mq_request_completed(struct request *rq)  in blk_mq_request_completed()  argument
810 	return blk_mq_rq_state(rq) == MQ_RQ_COMPLETE;  in blk_mq_request_completed()
820 static inline void blk_mq_set_request_complete(struct request *rq)  in blk_mq_set_request_complete()  argument
822 	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);  in blk_mq_set_request_complete()
829 static inline void blk_mq_complete_request_direct(struct request *rq,  in blk_mq_complete_request_direct()  argument
830 		   void (*complete)(struct request *rq))  in blk_mq_complete_request_direct()  argument
832 	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);  in blk_mq_complete_request_direct()
833 	complete(rq);  in blk_mq_complete_request_direct()
836 void blk_mq_start_request(struct request *rq);
837 void blk_mq_end_request(struct request *rq, blk_status_t error);
838 void __blk_mq_end_request(struct request *rq, blk_status_t error);
845 static inline bool blk_mq_need_time_stamp(struct request *rq)  in blk_mq_need_time_stamp()  argument
847 	return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_USE_SCHED));  in blk_mq_need_time_stamp()
850 static inline bool blk_mq_is_reserved_rq(struct request *rq)  in blk_mq_is_reserved_rq()  argument
852 	return rq->rq_flags & RQF_RESV;  in blk_mq_is_reserved_rq()
898 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list);
901 void blk_mq_complete_request(struct request *rq);
902 bool blk_mq_complete_request_remote(struct request *rq);
951 unsigned int blk_mq_rq_cpu(struct request *rq);
985 static inline void *blk_mq_rq_to_pdu(struct request *rq)  in blk_mq_rq_to_pdu()  argument
987 	return rq + 1;  in blk_mq_rq_to_pdu()
997 static inline void blk_mq_cleanup_rq(struct request *rq)  in blk_mq_cleanup_rq()  argument
999 	if (rq->q->mq_ops->cleanup_rq)  in blk_mq_cleanup_rq()
1000 		rq->q->mq_ops->cleanup_rq(rq);  in blk_mq_cleanup_rq()
1006 static inline bool rq_is_sync(struct request *rq)  in rq_is_sync()  argument
1008 	return op_is_sync(rq->cmd_flags);  in rq_is_sync()
1011 void blk_rq_init(struct request_queue *q, struct request *rq);
1012 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
1015 void blk_rq_unprep_clone(struct request *rq);
1016 blk_status_t blk_insert_cloned_request(struct request *rq);
1036 int blk_rq_append_bio(struct request *rq, struct bio *bio);
1037 void blk_execute_rq_nowait(struct request *rq, bool at_head);
1038 blk_status_t blk_execute_rq(struct request *rq, bool at_head);
1039 bool blk_rq_is_poll(struct request *rq);
1046 #define __rq_for_each_bio(_bio, rq)	\  argument
1047 	if ((rq->bio))			\
1048 		for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
1070 static inline sector_t blk_rq_pos(const struct request *rq)  in blk_rq_pos()  argument
1072 	return rq->__sector;  in blk_rq_pos()
1075 static inline unsigned int blk_rq_bytes(const struct request *rq)  in blk_rq_bytes()  argument
1077 	return rq->__data_len;  in blk_rq_bytes()
1080 static inline int blk_rq_cur_bytes(const struct request *rq)  in blk_rq_cur_bytes()  argument
1082 	if (!rq->bio)  in blk_rq_cur_bytes()
1084 	if (!bio_has_data(rq->bio))	/* dataless requests such as discard */  in blk_rq_cur_bytes()
1085 		return rq->bio->bi_iter.bi_size;  in blk_rq_cur_bytes()
1086 	return bio_iovec(rq->bio).bv_len;  in blk_rq_cur_bytes()
1089 static inline unsigned int blk_rq_sectors(const struct request *rq)  in blk_rq_sectors()  argument
1091 	return blk_rq_bytes(rq) >> SECTOR_SHIFT;  in blk_rq_sectors()
1094 static inline unsigned int blk_rq_cur_sectors(const struct request *rq)  in blk_rq_cur_sectors()  argument
1096 	return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;  in blk_rq_cur_sectors()
1099 static inline unsigned int blk_rq_stats_sectors(const struct request *rq)  in blk_rq_stats_sectors()  argument
1101 	return rq->stats_sectors;  in blk_rq_stats_sectors()
1110 static inline unsigned int blk_rq_payload_bytes(struct request *rq)  in blk_rq_payload_bytes()  argument
1112 	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)  in blk_rq_payload_bytes()
1113 		return rq->special_vec.bv_len;  in blk_rq_payload_bytes()
1114 	return blk_rq_bytes(rq);  in blk_rq_payload_bytes()
1121 static inline struct bio_vec req_bvec(struct request *rq)  in req_bvec()  argument
1123 	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)  in req_bvec()
1124 		return rq->special_vec;  in req_bvec()
1125 	return mp_bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter);  in req_bvec()
1128 static inline unsigned int blk_rq_count_bios(struct request *rq)  in blk_rq_count_bios()  argument
1133 	__rq_for_each_bio(bio, rq)  in blk_rq_count_bios()
1139 void blk_steal_bios(struct bio_list *list, struct request *rq);
1147 bool blk_update_request(struct request *rq, blk_status_t error,
1160 static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)  in blk_rq_nr_phys_segments()  argument
1162 	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)  in blk_rq_nr_phys_segments()
1164 	return rq->nr_phys_segments;  in blk_rq_nr_phys_segments()
1171 static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)  in blk_rq_nr_discard_segments()  argument
1173 	return max_t(unsigned short, rq->nr_phys_segments, 1);  in blk_rq_nr_discard_segments()
1176 int __blk_rq_map_sg(struct request *rq, struct scatterlist *sglist,
1178 static inline int blk_rq_map_sg(struct request *rq, struct scatterlist *sglist)  in blk_rq_map_sg()  argument
1182 	return __blk_rq_map_sg(rq, sglist, &last_sg);  in blk_rq_map_sg()