62 	struct request_queue *q = rq->q;  in elv_iosched_allow_bio_merge()  local
63 	struct elevator_queue *e = q->elevator;  in elv_iosched_allow_bio_merge()
66 		return e->type->ops.allow_merge(q, rq, bio);  in elv_iosched_allow_bio_merge()
123 struct elevator_queue *elevator_alloc(struct request_queue *q,  in elevator_alloc()  argument
128 	eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);  in elevator_alloc()
151 void elevator_exit(struct request_queue *q)  in elevator_exit()  argument
153 	struct elevator_queue *e = q->elevator;  in elevator_exit()
155 	ioc_clear_queue(q);  in elevator_exit()
156 	blk_mq_sched_free_rqs(q);  in elevator_exit()
159 	blk_mq_exit_sched(q, e);  in elevator_exit()
171 void elv_rqhash_del(struct request_queue *q, struct request *rq)  in elv_rqhash_del()  argument
178 void elv_rqhash_add(struct request_queue *q, struct request *rq)  in elv_rqhash_add()  argument
180 	struct elevator_queue *e = q->elevator;  in elv_rqhash_add()
188 void elv_rqhash_reposition(struct request_queue *q, struct request *rq)  in elv_rqhash_reposition()  argument
191 	elv_rqhash_add(q, rq);  in elv_rqhash_reposition()
194 struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)  in elv_rqhash_find()  argument
196 	struct elevator_queue *e = q->elevator;  in elv_rqhash_find()
268 enum elv_merge elv_merge(struct request_queue *q, struct request **req,  in elv_merge()  argument
271 	struct elevator_queue *e = q->elevator;  in elv_merge()
280 	if (blk_queue_nomerges(q) || !bio_mergeable(bio))  in elv_merge()
286 	if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) {  in elv_merge()
287 		enum elv_merge ret = blk_try_merge(q->last_merge, bio);  in elv_merge()
290 			*req = q->last_merge;  in elv_merge()
295 	if (blk_queue_noxmerges(q))  in elv_merge()
301 	__rq = elv_rqhash_find(q, bio->bi_iter.bi_sector);  in elv_merge()
311 		return e->type->ops.request_merge(q, req, bio);  in elv_merge()
324 bool elv_attempt_insert_merge(struct request_queue *q, struct request *rq,  in elv_attempt_insert_merge()  argument
330 	if (blk_queue_nomerges(q))  in elv_attempt_insert_merge()
336 	if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq)) {  in elv_attempt_insert_merge()
341 	if (blk_queue_noxmerges(q))  in elv_attempt_insert_merge()
349 		__rq = elv_rqhash_find(q, blk_rq_pos(rq));  in elv_attempt_insert_merge()
350 		if (!__rq || !blk_attempt_req_merge(q, __rq, rq))  in elv_attempt_insert_merge()
362 void elv_merged_request(struct request_queue *q, struct request *rq,  in elv_merged_request()  argument
365 	struct elevator_queue *e = q->elevator;  in elv_merged_request()
368 		e->type->ops.request_merged(q, rq, type);  in elv_merged_request()
371 		elv_rqhash_reposition(q, rq);  in elv_merged_request()
373 	q->last_merge = rq;  in elv_merged_request()
376 void elv_merge_requests(struct request_queue *q, struct request *rq,  in elv_merge_requests()  argument
379 	struct elevator_queue *e = q->elevator;  in elv_merge_requests()
382 		e->type->ops.requests_merged(q, rq, next);  in elv_merge_requests()
384 	elv_rqhash_reposition(q, rq);  in elv_merge_requests()
385 	q->last_merge = rq;  in elv_merge_requests()
388 struct request *elv_latter_request(struct request_queue *q, struct request *rq)  in elv_latter_request()  argument
390 	struct elevator_queue *e = q->elevator;  in elv_latter_request()
393 		return e->type->ops.next_request(q, rq);  in elv_latter_request()
398 struct request *elv_former_request(struct request_queue *q, struct request *rq)  in elv_former_request()  argument
400 	struct elevator_queue *e = q->elevator;  in elv_former_request()
403 		return e->type->ops.former_request(q, rq);  in elv_former_request()
455 int elv_register_queue(struct request_queue *q, bool uevent)  in elv_register_queue()  argument
457 	struct elevator_queue *e = q->elevator;  in elv_register_queue()
460 	lockdep_assert_held(&q->elevator_lock);  in elv_register_queue()
462 	error = kobject_add(&e->kobj, &q->disk->queue_kobj, "iosched");  in elv_register_queue()
480 void elv_unregister_queue(struct request_queue *q)  in elv_unregister_queue()  argument
482 	struct elevator_queue *e = q->elevator;  in elv_unregister_queue()
484 	lockdep_assert_held(&q->elevator_lock);  in elv_unregister_queue()
554 static struct elevator_type *elevator_get_default(struct request_queue *q)  in elevator_get_default()  argument
556 	if (q->tag_set->flags & BLK_MQ_F_NO_SCHED_BY_DEFAULT)  in elevator_get_default()
559 	if (q->nr_hw_queues != 1 &&  in elevator_get_default()
560 	    !blk_mq_is_shared_tags(q->tag_set->flags))  in elevator_get_default()
570 void elevator_init_mq(struct request_queue *q)  in elevator_init_mq()  argument
576 	WARN_ON_ONCE(blk_queue_registered(q));  in elevator_init_mq()
578 	if (unlikely(q->elevator))  in elevator_init_mq()
581 	e = elevator_get_default(q);  in elevator_init_mq()
594 	memflags = blk_mq_freeze_queue(q);  in elevator_init_mq()
596 	blk_mq_cancel_work_sync(q);  in elevator_init_mq()
598 	err = blk_mq_init_sched(q, e);  in elevator_init_mq()
600 	blk_mq_unfreeze_queue(q, memflags);  in elevator_init_mq()
616 int elevator_switch(struct request_queue *q, struct elevator_type *new_e)  in elevator_switch()  argument
621 	lockdep_assert_held(&q->elevator_lock);  in elevator_switch()
623 	memflags = blk_mq_freeze_queue(q);  in elevator_switch()
624 	blk_mq_quiesce_queue(q);  in elevator_switch()
626 	if (q->elevator) {  in elevator_switch()
627 		elv_unregister_queue(q);  in elevator_switch()
628 		elevator_exit(q);  in elevator_switch()
631 	ret = blk_mq_init_sched(q, new_e);  in elevator_switch()
635 	ret = elv_register_queue(q, true);  in elevator_switch()
637 		elevator_exit(q);  in elevator_switch()
640 	blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);  in elevator_switch()
643 	blk_mq_unquiesce_queue(q);  in elevator_switch()
644 	blk_mq_unfreeze_queue(q, memflags);  in elevator_switch()
654 void elevator_disable(struct request_queue *q)  in elevator_disable()  argument
658 	lockdep_assert_held(&q->elevator_lock);  in elevator_disable()
660 	memflags = blk_mq_freeze_queue(q);  in elevator_disable()
661 	blk_mq_quiesce_queue(q);  in elevator_disable()
663 	elv_unregister_queue(q);  in elevator_disable()
664 	elevator_exit(q);  in elevator_disable()
665 	blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q);  in elevator_disable()
666 	q->elevator = NULL;  in elevator_disable()
667 	q->nr_requests = q->tag_set->queue_depth;  in elevator_disable()
668 	blk_add_trace_msg(q, "elv switch: none");  in elevator_disable()
670 	blk_mq_unquiesce_queue(q);  in elevator_disable()
671 	blk_mq_unfreeze_queue(q, memflags);  in elevator_disable()
677 static int elevator_change(struct request_queue *q, const char *elevator_name)  in elevator_change()  argument
683 	if (!blk_queue_registered(q))  in elevator_change()
687 		if (q->elevator)  in elevator_change()
688 			elevator_disable(q);  in elevator_change()
692 	if (q->elevator && elevator_match(q->elevator->type, elevator_name))  in elevator_change()
698 	ret = elevator_switch(q, e);  in elevator_change()
722 	struct request_queue *q = disk->queue;  in elv_iosched_store()  local
734 	memflags = blk_mq_freeze_queue(q);  in elv_iosched_store()
735 	mutex_lock(&q->elevator_lock);  in elv_iosched_store()
736 	ret = elevator_change(q, name);  in elv_iosched_store()
739 	mutex_unlock(&q->elevator_lock);  in elv_iosched_store()
740 	blk_mq_unfreeze_queue(q, memflags);  in elv_iosched_store()
746 	struct request_queue *q = disk->queue;  in elv_iosched_show()  local
747 	struct elevator_queue *eq = q->elevator;  in elv_iosched_show()
751 	mutex_lock(&q->elevator_lock);  in elv_iosched_show()
752 	if (!q->elevator) {  in elv_iosched_show()
769 	mutex_unlock(&q->elevator_lock);  in elv_iosched_show()
774 struct request *elv_rb_former_request(struct request_queue *q,  in elv_rb_former_request()  argument
786 struct request *elv_rb_latter_request(struct request_queue *q,  in elv_rb_latter_request()  argument