1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef BLK_MQ_H 3 #define BLK_MQ_H 4 5 #include <linux/blkdev.h> 6 #include <linux/sbitmap.h> 7 #include <linux/srcu.h> 8 9 struct blk_mq_tags; 10 struct blk_flush_queue; 11 12 /** 13 * struct blk_mq_hw_ctx - State for a hardware queue facing the hardware block device 14 */ 15 struct blk_mq_hw_ctx { 16 struct { 17 spinlock_t lock; 18 struct list_head dispatch; 19 unsigned long state; /* BLK_MQ_S_* flags */ 20 } ____cacheline_aligned_in_smp; 21 22 struct delayed_work run_work; 23 cpumask_var_t cpumask; 24 int next_cpu; 25 int next_cpu_batch; 26 27 unsigned long flags; /* BLK_MQ_F_* flags */ 28 29 void *sched_data; 30 struct request_queue *queue; 31 struct blk_flush_queue *fq; 32 33 void *driver_data; 34 35 struct sbitmap ctx_map; 36 37 struct blk_mq_ctx *dispatch_from; 38 unsigned int dispatch_busy; 39 40 unsigned short type; 41 unsigned short nr_ctx; 42 struct blk_mq_ctx **ctxs; 43 44 spinlock_t dispatch_wait_lock; 45 wait_queue_entry_t dispatch_wait; 46 atomic_t wait_index; 47 48 struct blk_mq_tags *tags; 49 struct blk_mq_tags *sched_tags; 50 51 unsigned long queued; 52 unsigned long run; 53 #define BLK_MQ_MAX_DISPATCH_ORDER 7 54 unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER]; 55 56 unsigned int numa_node; 57 unsigned int queue_num; 58 59 atomic_t nr_active; 60 61 struct hlist_node cpuhp_dead; 62 struct kobject kobj; 63 64 unsigned long poll_considered; 65 unsigned long poll_invoked; 66 unsigned long poll_success; 67 68 #ifdef CONFIG_BLK_DEBUG_FS 69 struct dentry *debugfs_dir; 70 struct dentry *sched_debugfs_dir; 71 #endif 72 73 /* Must be the last member - see also blk_mq_hw_ctx_size(). */ 74 struct srcu_struct srcu[0]; 75 }; 76 77 struct blk_mq_queue_map { 78 unsigned int *mq_map; 79 unsigned int nr_queues; 80 unsigned int queue_offset; 81 }; 82 83 enum hctx_type { 84 HCTX_TYPE_DEFAULT, /* all I/O not otherwise accounted for */ 85 HCTX_TYPE_READ, /* just for READ I/O */ 86 HCTX_TYPE_POLL, /* polled I/O of any kind */ 87 88 HCTX_MAX_TYPES, 89 }; 90 91 struct blk_mq_tag_set { 92 /* 93 * map[] holds ctx -> hctx mappings, one map exists for each type 94 * that the driver wishes to support. There are no restrictions 95 * on maps being of the same size, and it's perfectly legal to 96 * share maps between types. 97 */ 98 struct blk_mq_queue_map map[HCTX_MAX_TYPES]; 99 unsigned int nr_maps; /* nr entries in map[] */ 100 const struct blk_mq_ops *ops; 101 unsigned int nr_hw_queues; /* nr hw queues across maps */ 102 unsigned int queue_depth; /* max hw supported */ 103 unsigned int reserved_tags; 104 unsigned int cmd_size; /* per-request extra data */ 105 int numa_node; 106 unsigned int timeout; 107 unsigned int flags; /* BLK_MQ_F_* */ 108 void *driver_data; 109 110 struct blk_mq_tags **tags; 111 112 struct mutex tag_list_lock; 113 struct list_head tag_list; 114 }; 115 116 struct blk_mq_queue_data { 117 struct request *rq; 118 bool last; 119 }; 120 121 typedef blk_status_t (queue_rq_fn)(struct blk_mq_hw_ctx *, 122 const struct blk_mq_queue_data *); 123 typedef void (commit_rqs_fn)(struct blk_mq_hw_ctx *); 124 typedef bool (get_budget_fn)(struct blk_mq_hw_ctx *); 125 typedef void (put_budget_fn)(struct blk_mq_hw_ctx *); 126 typedef enum blk_eh_timer_return (timeout_fn)(struct request *, bool); 127 typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int); 128 typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int); 129 typedef int (init_request_fn)(struct blk_mq_tag_set *set, struct request *, 130 unsigned int, unsigned int); 131 typedef void (exit_request_fn)(struct blk_mq_tag_set *set, struct request *, 132 unsigned int); 133 134 typedef bool (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *, 135 bool); 136 typedef bool (busy_tag_iter_fn)(struct request *, void *, bool); 137 typedef int (poll_fn)(struct blk_mq_hw_ctx *); 138 typedef int (map_queues_fn)(struct blk_mq_tag_set *set); 139 typedef bool (busy_fn)(struct request_queue *); 140 typedef void (complete_fn)(struct request *); 141 142 143 struct blk_mq_ops { 144 /* 145 * Queue request 146 */ 147 queue_rq_fn *queue_rq; 148 149 /* 150 * If a driver uses bd->last to judge when to submit requests to 151 * hardware, it must define this function. In case of errors that 152 * make us stop issuing further requests, this hook serves the 153 * purpose of kicking the hardware (which the last request otherwise 154 * would have done). 155 */ 156 commit_rqs_fn *commit_rqs; 157 158 /* 159 * Reserve budget before queue request, once .queue_rq is 160 * run, it is driver's responsibility to release the 161 * reserved budget. Also we have to handle failure case 162 * of .get_budget for avoiding I/O deadlock. 163 */ 164 get_budget_fn *get_budget; 165 put_budget_fn *put_budget; 166 167 /* 168 * Called on request timeout 169 */ 170 timeout_fn *timeout; 171 172 /* 173 * Called to poll for completion of a specific tag. 174 */ 175 poll_fn *poll; 176 177 complete_fn *complete; 178 179 /* 180 * Called when the block layer side of a hardware queue has been 181 * set up, allowing the driver to allocate/init matching structures. 182 * Ditto for exit/teardown. 183 */ 184 init_hctx_fn *init_hctx; 185 exit_hctx_fn *exit_hctx; 186 187 /* 188 * Called for every command allocated by the block layer to allow 189 * the driver to set up driver specific data. 190 * 191 * Tag greater than or equal to queue_depth is for setting up 192 * flush request. 193 * 194 * Ditto for exit/teardown. 195 */ 196 init_request_fn *init_request; 197 exit_request_fn *exit_request; 198 /* Called from inside blk_get_request() */ 199 void (*initialize_rq_fn)(struct request *rq); 200 201 /* 202 * If set, returns whether or not this queue currently is busy 203 */ 204 busy_fn *busy; 205 206 map_queues_fn *map_queues; 207 208 #ifdef CONFIG_BLK_DEBUG_FS 209 /* 210 * Used by the debugfs implementation to show driver-specific 211 * information about a request. 212 */ 213 void (*show_rq)(struct seq_file *m, struct request *rq); 214 #endif 215 }; 216 217 enum { 218 BLK_MQ_F_SHOULD_MERGE = 1 << 0, 219 BLK_MQ_F_TAG_SHARED = 1 << 1, 220 BLK_MQ_F_BLOCKING = 1 << 5, 221 BLK_MQ_F_NO_SCHED = 1 << 6, 222 BLK_MQ_F_ALLOC_POLICY_START_BIT = 8, 223 BLK_MQ_F_ALLOC_POLICY_BITS = 1, 224 225 BLK_MQ_S_STOPPED = 0, 226 BLK_MQ_S_TAG_ACTIVE = 1, 227 BLK_MQ_S_SCHED_RESTART = 2, 228 229 BLK_MQ_MAX_DEPTH = 10240, 230 231 BLK_MQ_CPU_WORK_BATCH = 8, 232 }; 233 #define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \ 234 ((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \ 235 ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) 236 #define BLK_ALLOC_POLICY_TO_MQ_FLAG(policy) \ 237 ((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \ 238 << BLK_MQ_F_ALLOC_POLICY_START_BIT) 239 240 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); 241 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 242 struct request_queue *q); 243 struct request_queue *blk_mq_init_sq_queue(struct blk_mq_tag_set *set, 244 const struct blk_mq_ops *ops, 245 unsigned int queue_depth, 246 unsigned int set_flags); 247 int blk_mq_register_dev(struct device *, struct request_queue *); 248 void blk_mq_unregister_dev(struct device *, struct request_queue *); 249 250 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set); 251 void blk_mq_free_tag_set(struct blk_mq_tag_set *set); 252 253 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule); 254 255 void blk_mq_free_request(struct request *rq); 256 bool blk_mq_can_queue(struct blk_mq_hw_ctx *); 257 258 bool blk_mq_queue_inflight(struct request_queue *q); 259 260 enum { 261 /* return when out of requests */ 262 BLK_MQ_REQ_NOWAIT = (__force blk_mq_req_flags_t)(1 << 0), 263 /* allocate from reserved pool */ 264 BLK_MQ_REQ_RESERVED = (__force blk_mq_req_flags_t)(1 << 1), 265 /* allocate internal/sched tag */ 266 BLK_MQ_REQ_INTERNAL = (__force blk_mq_req_flags_t)(1 << 2), 267 /* set RQF_PREEMPT */ 268 BLK_MQ_REQ_PREEMPT = (__force blk_mq_req_flags_t)(1 << 3), 269 }; 270 271 struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, 272 blk_mq_req_flags_t flags); 273 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, 274 unsigned int op, blk_mq_req_flags_t flags, 275 unsigned int hctx_idx); 276 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag); 277 278 enum { 279 BLK_MQ_UNIQUE_TAG_BITS = 16, 280 BLK_MQ_UNIQUE_TAG_MASK = (1 << BLK_MQ_UNIQUE_TAG_BITS) - 1, 281 }; 282 283 u32 blk_mq_unique_tag(struct request *rq); 284 285 static inline u16 blk_mq_unique_tag_to_hwq(u32 unique_tag) 286 { 287 return unique_tag >> BLK_MQ_UNIQUE_TAG_BITS; 288 } 289 290 static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag) 291 { 292 return unique_tag & BLK_MQ_UNIQUE_TAG_MASK; 293 } 294 295 296 int blk_mq_request_started(struct request *rq); 297 void blk_mq_start_request(struct request *rq); 298 void blk_mq_end_request(struct request *rq, blk_status_t error); 299 void __blk_mq_end_request(struct request *rq, blk_status_t error); 300 301 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list); 302 void blk_mq_kick_requeue_list(struct request_queue *q); 303 void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs); 304 bool blk_mq_complete_request(struct request *rq); 305 bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list, 306 struct bio *bio); 307 bool blk_mq_queue_stopped(struct request_queue *q); 308 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx); 309 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx); 310 void blk_mq_stop_hw_queues(struct request_queue *q); 311 void blk_mq_start_hw_queues(struct request_queue *q); 312 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); 313 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async); 314 void blk_mq_quiesce_queue(struct request_queue *q); 315 void blk_mq_unquiesce_queue(struct request_queue *q); 316 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); 317 bool blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); 318 void blk_mq_run_hw_queues(struct request_queue *q, bool async); 319 void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, 320 busy_tag_iter_fn *fn, void *priv); 321 void blk_mq_freeze_queue(struct request_queue *q); 322 void blk_mq_unfreeze_queue(struct request_queue *q); 323 void blk_freeze_queue_start(struct request_queue *q); 324 void blk_mq_freeze_queue_wait(struct request_queue *q); 325 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, 326 unsigned long timeout); 327 328 int blk_mq_map_queues(struct blk_mq_queue_map *qmap); 329 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues); 330 331 void blk_mq_quiesce_queue_nowait(struct request_queue *q); 332 333 unsigned int blk_mq_rq_cpu(struct request *rq); 334 335 /* 336 * Driver command data is immediately after the request. So subtract request 337 * size to get back to the original request, add request size to get the PDU. 338 */ 339 static inline struct request *blk_mq_rq_from_pdu(void *pdu) 340 { 341 return pdu - sizeof(struct request); 342 } 343 static inline void *blk_mq_rq_to_pdu(struct request *rq) 344 { 345 return rq + 1; 346 } 347 348 #define queue_for_each_hw_ctx(q, hctx, i) \ 349 for ((i) = 0; (i) < (q)->nr_hw_queues && \ 350 ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++) 351 352 #define hctx_for_each_ctx(hctx, ctx, i) \ 353 for ((i) = 0; (i) < (hctx)->nr_ctx && \ 354 ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) 355 356 static inline blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, 357 struct request *rq) 358 { 359 if (rq->tag != -1) 360 return rq->tag | (hctx->queue_num << BLK_QC_T_SHIFT); 361 362 return rq->internal_tag | (hctx->queue_num << BLK_QC_T_SHIFT) | 363 BLK_QC_T_INTERNAL; 364 } 365 366 #endif 367