1 #ifndef BLK_MQ_H 2 #define BLK_MQ_H 3 4 #include <linux/blkdev.h> 5 #include <linux/sbitmap.h> 6 7 struct blk_mq_tags; 8 struct blk_flush_queue; 9 10 struct blk_mq_hw_ctx { 11 struct { 12 spinlock_t lock; 13 struct list_head dispatch; 14 unsigned long state; /* BLK_MQ_S_* flags */ 15 } ____cacheline_aligned_in_smp; 16 17 struct work_struct run_work; 18 cpumask_var_t cpumask; 19 int next_cpu; 20 int next_cpu_batch; 21 22 unsigned long flags; /* BLK_MQ_F_* flags */ 23 24 struct request_queue *queue; 25 struct blk_flush_queue *fq; 26 27 void *driver_data; 28 29 struct sbitmap ctx_map; 30 31 struct blk_mq_ctx **ctxs; 32 unsigned int nr_ctx; 33 34 atomic_t wait_index; 35 36 struct blk_mq_tags *tags; 37 38 unsigned long queued; 39 unsigned long run; 40 #define BLK_MQ_MAX_DISPATCH_ORDER 7 41 unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER]; 42 43 unsigned int numa_node; 44 unsigned int queue_num; 45 46 atomic_t nr_active; 47 48 struct delayed_work delay_work; 49 50 struct hlist_node cpuhp_dead; 51 struct kobject kobj; 52 53 unsigned long poll_considered; 54 unsigned long poll_invoked; 55 unsigned long poll_success; 56 }; 57 58 struct blk_mq_tag_set { 59 unsigned int *mq_map; 60 struct blk_mq_ops *ops; 61 unsigned int nr_hw_queues; 62 unsigned int queue_depth; /* max hw supported */ 63 unsigned int reserved_tags; 64 unsigned int cmd_size; /* per-request extra data */ 65 int numa_node; 66 unsigned int timeout; 67 unsigned int flags; /* BLK_MQ_F_* */ 68 void *driver_data; 69 70 struct blk_mq_tags **tags; 71 72 struct mutex tag_list_lock; 73 struct list_head tag_list; 74 }; 75 76 struct blk_mq_queue_data { 77 struct request *rq; 78 struct list_head *list; 79 bool last; 80 }; 81 82 typedef int (queue_rq_fn)(struct blk_mq_hw_ctx *, const struct blk_mq_queue_data *); 83 typedef enum blk_eh_timer_return (timeout_fn)(struct request *, bool); 84 typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int); 85 typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int); 86 typedef int (init_request_fn)(void *, struct request *, unsigned int, 87 unsigned int, unsigned int); 88 typedef void (exit_request_fn)(void *, struct request *, unsigned int, 89 unsigned int); 90 typedef int (reinit_request_fn)(void *, struct request *); 91 92 typedef void (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *, 93 bool); 94 typedef void (busy_tag_iter_fn)(struct request *, void *, bool); 95 typedef int (poll_fn)(struct blk_mq_hw_ctx *, unsigned int); 96 typedef int (map_queues_fn)(struct blk_mq_tag_set *set); 97 98 99 struct blk_mq_ops { 100 /* 101 * Queue request 102 */ 103 queue_rq_fn *queue_rq; 104 105 /* 106 * Called on request timeout 107 */ 108 timeout_fn *timeout; 109 110 /* 111 * Called to poll for completion of a specific tag. 112 */ 113 poll_fn *poll; 114 115 softirq_done_fn *complete; 116 117 /* 118 * Called when the block layer side of a hardware queue has been 119 * set up, allowing the driver to allocate/init matching structures. 120 * Ditto for exit/teardown. 121 */ 122 init_hctx_fn *init_hctx; 123 exit_hctx_fn *exit_hctx; 124 125 /* 126 * Called for every command allocated by the block layer to allow 127 * the driver to set up driver specific data. 128 * 129 * Tag greater than or equal to queue_depth is for setting up 130 * flush request. 131 * 132 * Ditto for exit/teardown. 133 */ 134 init_request_fn *init_request; 135 exit_request_fn *exit_request; 136 reinit_request_fn *reinit_request; 137 138 map_queues_fn *map_queues; 139 }; 140 141 enum { 142 BLK_MQ_RQ_QUEUE_OK = 0, /* queued fine */ 143 BLK_MQ_RQ_QUEUE_BUSY = 1, /* requeue IO for later */ 144 BLK_MQ_RQ_QUEUE_ERROR = 2, /* end IO with error */ 145 146 BLK_MQ_F_SHOULD_MERGE = 1 << 0, 147 BLK_MQ_F_TAG_SHARED = 1 << 1, 148 BLK_MQ_F_SG_MERGE = 1 << 2, 149 BLK_MQ_F_DEFER_ISSUE = 1 << 4, 150 BLK_MQ_F_BLOCKING = 1 << 5, 151 BLK_MQ_F_ALLOC_POLICY_START_BIT = 8, 152 BLK_MQ_F_ALLOC_POLICY_BITS = 1, 153 154 BLK_MQ_S_STOPPED = 0, 155 BLK_MQ_S_TAG_ACTIVE = 1, 156 157 BLK_MQ_MAX_DEPTH = 10240, 158 159 BLK_MQ_CPU_WORK_BATCH = 8, 160 }; 161 #define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \ 162 ((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \ 163 ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) 164 #define BLK_ALLOC_POLICY_TO_MQ_FLAG(policy) \ 165 ((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \ 166 << BLK_MQ_F_ALLOC_POLICY_START_BIT) 167 168 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); 169 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 170 struct request_queue *q); 171 int blk_mq_register_dev(struct device *, struct request_queue *); 172 void blk_mq_unregister_dev(struct device *, struct request_queue *); 173 174 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set); 175 void blk_mq_free_tag_set(struct blk_mq_tag_set *set); 176 177 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule); 178 179 void blk_mq_insert_request(struct request *, bool, bool, bool); 180 void blk_mq_free_request(struct request *rq); 181 void blk_mq_free_hctx_request(struct blk_mq_hw_ctx *, struct request *rq); 182 bool blk_mq_can_queue(struct blk_mq_hw_ctx *); 183 184 enum { 185 BLK_MQ_REQ_NOWAIT = (1 << 0), /* return when out of requests */ 186 BLK_MQ_REQ_RESERVED = (1 << 1), /* allocate from reserved pool */ 187 }; 188 189 struct request *blk_mq_alloc_request(struct request_queue *q, int rw, 190 unsigned int flags); 191 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int op, 192 unsigned int flags, unsigned int hctx_idx); 193 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag); 194 195 enum { 196 BLK_MQ_UNIQUE_TAG_BITS = 16, 197 BLK_MQ_UNIQUE_TAG_MASK = (1 << BLK_MQ_UNIQUE_TAG_BITS) - 1, 198 }; 199 200 u32 blk_mq_unique_tag(struct request *rq); 201 202 static inline u16 blk_mq_unique_tag_to_hwq(u32 unique_tag) 203 { 204 return unique_tag >> BLK_MQ_UNIQUE_TAG_BITS; 205 } 206 207 static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag) 208 { 209 return unique_tag & BLK_MQ_UNIQUE_TAG_MASK; 210 } 211 212 213 int blk_mq_request_started(struct request *rq); 214 void blk_mq_start_request(struct request *rq); 215 void blk_mq_end_request(struct request *rq, int error); 216 void __blk_mq_end_request(struct request *rq, int error); 217 218 void blk_mq_requeue_request(struct request *rq); 219 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head); 220 void blk_mq_cancel_requeue_work(struct request_queue *q); 221 void blk_mq_kick_requeue_list(struct request_queue *q); 222 void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs); 223 void blk_mq_abort_requeue_list(struct request_queue *q); 224 void blk_mq_complete_request(struct request *rq, int error); 225 226 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx); 227 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx); 228 void blk_mq_stop_hw_queues(struct request_queue *q); 229 void blk_mq_start_hw_queues(struct request_queue *q); 230 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async); 231 void blk_mq_run_hw_queues(struct request_queue *q, bool async); 232 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); 233 void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, 234 busy_tag_iter_fn *fn, void *priv); 235 void blk_mq_freeze_queue(struct request_queue *q); 236 void blk_mq_unfreeze_queue(struct request_queue *q); 237 void blk_mq_freeze_queue_start(struct request_queue *q); 238 int blk_mq_reinit_tagset(struct blk_mq_tag_set *set); 239 240 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues); 241 242 /* 243 * Driver command data is immediately after the request. So subtract request 244 * size to get back to the original request, add request size to get the PDU. 245 */ 246 static inline struct request *blk_mq_rq_from_pdu(void *pdu) 247 { 248 return pdu - sizeof(struct request); 249 } 250 static inline void *blk_mq_rq_to_pdu(struct request *rq) 251 { 252 return rq + 1; 253 } 254 255 #define queue_for_each_hw_ctx(q, hctx, i) \ 256 for ((i) = 0; (i) < (q)->nr_hw_queues && \ 257 ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++) 258 259 #define hctx_for_each_ctx(hctx, ctx, i) \ 260 for ((i) = 0; (i) < (hctx)->nr_ctx && \ 261 ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) 262 263 #endif 264