226 			       struct mmc_queue *mq);
296 	struct mmc_queue *mq;  in power_ro_lock_store()  local
307 	mq = &md->queue;  in power_ro_lock_store()
310 	req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_OUT, 0);  in power_ro_lock_store()
710 	struct mmc_queue *mq;  in mmc_blk_ioctl_cmd()  local
730 	mq = &md->queue;  in mmc_blk_ioctl_cmd()
731 	req = blk_mq_alloc_request(mq->queue,  in mmc_blk_ioctl_cmd()
761 	struct mmc_queue *mq;  in mmc_blk_ioctl_multi_cmd()  local
803 	mq = &md->queue;  in mmc_blk_ioctl_multi_cmd()
804 	req = blk_mq_alloc_request(mq->queue,  in mmc_blk_ioctl_multi_cmd()
1143 static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)  in mmc_blk_issue_drv_op()  argument
1146 	struct mmc_card *card = mq->card;  in mmc_blk_issue_drv_op()
1147 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_drv_op()
1213 static void mmc_blk_issue_erase_rq(struct mmc_queue *mq, struct request *req,  in mmc_blk_issue_erase_rq()  argument
1216 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_erase_rq()
1252 static void mmc_blk_issue_trim_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_issue_trim_rq()  argument
1254 	mmc_blk_issue_erase_rq(mq, req, MMC_BLK_TRIM, MMC_TRIM_ARG);  in mmc_blk_issue_trim_rq()
1257 static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_issue_discard_rq()  argument
1259 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_discard_rq()
1266 	mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, arg);  in mmc_blk_issue_discard_rq()
1269 static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,  in mmc_blk_issue_secdiscard_rq()  argument
1272 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_secdiscard_rq()
1340 static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)  in mmc_blk_issue_flush()  argument
1342 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_flush()
1427 static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq,  in mmc_blk_data_prep()  argument
1431 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_data_prep()
1499 			brq->data.blocks = queue_physical_block_size(mq->queue) >> 9;  in mmc_blk_data_prep()
1533 	brq->data.sg_len = mmc_queue_map_sg(mq, mqrq);  in mmc_blk_data_prep()
1563 static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_cqe_complete_rq()  argument
1568 	struct mmc_host *host = mq->card->host;  in mmc_blk_cqe_complete_rq()
1569 	enum mmc_issue_type issue_type = mmc_issue_type(mq, req);  in mmc_blk_cqe_complete_rq()
1593 	} else if (mq->in_recovery) {  in mmc_blk_cqe_complete_rq()
1599 	spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_cqe_complete_rq()
1601 	mq->in_flight[issue_type] -= 1;  in mmc_blk_cqe_complete_rq()
1603 	put_card = (mmc_tot_in_flight(mq) == 0);  in mmc_blk_cqe_complete_rq()
1605 	mmc_cqe_check_busy(mq);  in mmc_blk_cqe_complete_rq()
1607 	spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_cqe_complete_rq()
1609 	if (!mq->cqe_busy)  in mmc_blk_cqe_complete_rq()
1613 		mmc_put_card(mq->card, &mq->ctx);  in mmc_blk_cqe_complete_rq()
1616 void mmc_blk_cqe_recovery(struct mmc_queue *mq)  in mmc_blk_cqe_recovery()  argument
1618 	struct mmc_card *card = mq->card;  in mmc_blk_cqe_recovery()
1626 		mmc_blk_reset(mq->blkdata, host, MMC_BLK_CQE_RECOVERY);  in mmc_blk_cqe_recovery()
1627 	mmc_blk_reset_success(mq->blkdata, MMC_BLK_CQE_RECOVERY);  in mmc_blk_cqe_recovery()
1638 	struct mmc_queue *mq = q->queuedata;  in mmc_blk_cqe_req_done()  local
1644 	if (mq->in_recovery)  in mmc_blk_cqe_req_done()
1645 		mmc_blk_cqe_complete_rq(mq, req);  in mmc_blk_cqe_req_done()
1671 static int mmc_blk_cqe_issue_flush(struct mmc_queue *mq, struct request *req)  in mmc_blk_cqe_issue_flush()  argument
1683 	return mmc_blk_cqe_start_req(mq->card->host, mrq);  in mmc_blk_cqe_issue_flush()
1686 static int mmc_blk_hsq_issue_rw_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_hsq_issue_rw_rq()  argument
1689 	struct mmc_host *host = mq->card->host;  in mmc_blk_hsq_issue_rw_rq()
1692 	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);  in mmc_blk_hsq_issue_rw_rq()
1703 static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_cqe_issue_rw_rq()  argument
1706 	struct mmc_host *host = mq->card->host;  in mmc_blk_cqe_issue_rw_rq()
1709 		return mmc_blk_hsq_issue_rw_rq(mq, req);  in mmc_blk_cqe_issue_rw_rq()
1711 	mmc_blk_data_prep(mq, mqrq, 0, NULL, NULL);  in mmc_blk_cqe_issue_rw_rq()
1713 	return mmc_blk_cqe_start_req(mq->card->host, &mqrq->brq.mrq);  in mmc_blk_cqe_issue_rw_rq()
1719 			       struct mmc_queue *mq)  in mmc_blk_rw_rq_prep()  argument
1724 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_rw_rq_prep()
1727 	mmc_blk_data_prep(mq, mqrq, recovery_mode, &do_rel_wr, &do_data_tag);  in mmc_blk_rw_rq_prep()
1824 static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req)  in mmc_blk_read_single()  argument
1828 	struct mmc_card *card = mq->card;  in mmc_blk_read_single()
1831 	size_t bytes_per_read = queue_physical_block_size(mq->queue);  in mmc_blk_read_single()
1839 			mmc_blk_rw_rq_prep(mqrq, card, 1, mq);  in mmc_blk_read_single()
1896 	struct mmc_queue *mq = req->q->queuedata;  in mmc_blk_status_error()  local
1899 	if (mmc_host_is_spi(mq->card->host))  in mmc_blk_status_error()
1932 static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_rw_recovery()  argument
1937 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_mq_rw_recovery()
1938 	struct mmc_card *card = mq->card;  in mmc_blk_mq_rw_recovery()
1968 	if (!mmc_host_is_spi(mq->card->host) &&  in mmc_blk_mq_rw_recovery()
1970 		err = mmc_blk_fix_state(mq->card, req);  in mmc_blk_mq_rw_recovery()
1985 	if (!mmc_host_is_spi(mq->card->host) &&  in mmc_blk_mq_rw_recovery()
2015 			queue_physical_block_size(mq->queue) >> 9) {  in mmc_blk_mq_rw_recovery()
2017 		mmc_blk_read_single(mq, req);  in mmc_blk_mq_rw_recovery()
2109 static inline void mmc_blk_rw_reset_success(struct mmc_queue *mq,  in mmc_blk_rw_reset_success()  argument
2114 	mmc_blk_reset_success(mq->blkdata, type);  in mmc_blk_rw_reset_success()
2117 static void mmc_blk_mq_complete_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_complete_rq()  argument
2132 		if (mmc_card_removed(mq->card))  in mmc_blk_mq_complete_rq()
2138 static bool mmc_blk_urgent_bkops_needed(struct mmc_queue *mq,  in mmc_blk_urgent_bkops_needed()  argument
2141 	return mmc_card_mmc(mq->card) && !mmc_host_is_spi(mq->card->host) &&  in mmc_blk_urgent_bkops_needed()
2146 static void mmc_blk_urgent_bkops(struct mmc_queue *mq,  in mmc_blk_urgent_bkops()  argument
2149 	if (mmc_blk_urgent_bkops_needed(mq, mqrq))  in mmc_blk_urgent_bkops()
2150 		mmc_run_bkops(mq->card);  in mmc_blk_urgent_bkops()
2159 	struct mmc_queue *mq = q->queuedata;  in mmc_blk_hsq_req_done()  local
2160 	struct mmc_host *host = mq->card->host;  in mmc_blk_hsq_req_done()
2164 	    mmc_blk_urgent_bkops_needed(mq, mqrq)) {  in mmc_blk_hsq_req_done()
2165 		spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_hsq_req_done()
2166 		mq->recovery_needed = true;  in mmc_blk_hsq_req_done()
2167 		mq->recovery_req = req;  in mmc_blk_hsq_req_done()
2168 		spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_hsq_req_done()
2172 		schedule_work(&mq->recovery_work);  in mmc_blk_hsq_req_done()
2176 	mmc_blk_rw_reset_success(mq, req);  in mmc_blk_hsq_req_done()
2182 	if (mq->in_recovery)  in mmc_blk_hsq_req_done()
2183 		mmc_blk_cqe_complete_rq(mq, req);  in mmc_blk_hsq_req_done()
2190 	struct mmc_queue *mq = req->q->queuedata;  in mmc_blk_mq_complete()  local
2191 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_complete()
2194 		mmc_blk_cqe_complete_rq(mq, req);  in mmc_blk_mq_complete()
2196 		mmc_blk_mq_complete_rq(mq, req);  in mmc_blk_mq_complete()
2199 static void mmc_blk_mq_poll_completion(struct mmc_queue *mq,  in mmc_blk_mq_poll_completion()  argument
2203 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_poll_completion()
2206 	    mmc_blk_card_busy(mq->card, req)) {  in mmc_blk_mq_poll_completion()
2207 		mmc_blk_mq_rw_recovery(mq, req);  in mmc_blk_mq_poll_completion()
2209 		mmc_blk_rw_reset_success(mq, req);  in mmc_blk_mq_poll_completion()
2213 	mmc_blk_urgent_bkops(mq, mqrq);  in mmc_blk_mq_poll_completion()
2216 static void mmc_blk_mq_dec_in_flight(struct mmc_queue *mq, enum mmc_issue_type issue_type)  in mmc_blk_mq_dec_in_flight()  argument
2221 	spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_mq_dec_in_flight()
2223 	mq->in_flight[issue_type] -= 1;  in mmc_blk_mq_dec_in_flight()
2225 	put_card = (mmc_tot_in_flight(mq) == 0);  in mmc_blk_mq_dec_in_flight()
2227 	spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_mq_dec_in_flight()
2230 		mmc_put_card(mq->card, &mq->ctx);  in mmc_blk_mq_dec_in_flight()
2233 static void mmc_blk_mq_post_req(struct mmc_queue *mq, struct request *req,  in mmc_blk_mq_post_req()  argument
2236 	enum mmc_issue_type issue_type = mmc_issue_type(mq, req);  in mmc_blk_mq_post_req()
2239 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_post_req()
2247 	if (mq->in_recovery) {  in mmc_blk_mq_post_req()
2248 		mmc_blk_mq_complete_rq(mq, req);  in mmc_blk_mq_post_req()
2256 	mmc_blk_mq_dec_in_flight(mq, issue_type);  in mmc_blk_mq_post_req()
2259 void mmc_blk_mq_recovery(struct mmc_queue *mq)  in mmc_blk_mq_recovery()  argument
2261 	struct request *req = mq->recovery_req;  in mmc_blk_mq_recovery()
2262 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_recovery()
2265 	mq->recovery_req = NULL;  in mmc_blk_mq_recovery()
2266 	mq->rw_wait = false;  in mmc_blk_mq_recovery()
2270 		mmc_blk_mq_rw_recovery(mq, req);  in mmc_blk_mq_recovery()
2273 	mmc_blk_urgent_bkops(mq, mqrq);  in mmc_blk_mq_recovery()
2275 	mmc_blk_mq_post_req(mq, req, true);  in mmc_blk_mq_recovery()
2278 static void mmc_blk_mq_complete_prev_req(struct mmc_queue *mq,  in mmc_blk_mq_complete_prev_req()  argument
2281 	if (mmc_host_done_complete(mq->card->host))  in mmc_blk_mq_complete_prev_req()
2284 	mutex_lock(&mq->complete_lock);  in mmc_blk_mq_complete_prev_req()
2286 	if (!mq->complete_req)  in mmc_blk_mq_complete_prev_req()
2289 	mmc_blk_mq_poll_completion(mq, mq->complete_req);  in mmc_blk_mq_complete_prev_req()
2292 		*prev_req = mq->complete_req;  in mmc_blk_mq_complete_prev_req()
2294 		mmc_blk_mq_post_req(mq, mq->complete_req, true);  in mmc_blk_mq_complete_prev_req()
2296 	mq->complete_req = NULL;  in mmc_blk_mq_complete_prev_req()
2299 	mutex_unlock(&mq->complete_lock);  in mmc_blk_mq_complete_prev_req()
2304 	struct mmc_queue *mq = container_of(work, struct mmc_queue,  in mmc_blk_mq_complete_work()  local
2307 	mmc_blk_mq_complete_prev_req(mq, NULL);  in mmc_blk_mq_complete_work()
2316 	struct mmc_queue *mq = q->queuedata;  in mmc_blk_mq_req_done()  local
2317 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_req_done()
2329 		spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_mq_req_done()
2330 		mq->complete_req = req;  in mmc_blk_mq_req_done()
2331 		mq->rw_wait = false;  in mmc_blk_mq_req_done()
2332 		waiting = mq->waiting;  in mmc_blk_mq_req_done()
2333 		spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_mq_req_done()
2342 			wake_up(&mq->wait);  in mmc_blk_mq_req_done()
2344 			queue_work(mq->card->complete_wq, &mq->complete_work);  in mmc_blk_mq_req_done()
2351 	    mmc_blk_urgent_bkops_needed(mq, mqrq)) {  in mmc_blk_mq_req_done()
2352 		spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_mq_req_done()
2353 		mq->recovery_needed = true;  in mmc_blk_mq_req_done()
2354 		mq->recovery_req = req;  in mmc_blk_mq_req_done()
2355 		spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_mq_req_done()
2356 		wake_up(&mq->wait);  in mmc_blk_mq_req_done()
2357 		schedule_work(&mq->recovery_work);  in mmc_blk_mq_req_done()
2361 	mmc_blk_rw_reset_success(mq, req);  in mmc_blk_mq_req_done()
2363 	mq->rw_wait = false;  in mmc_blk_mq_req_done()
2364 	wake_up(&mq->wait);  in mmc_blk_mq_req_done()
2367 	mmc_blk_mq_post_req(mq, req, false);  in mmc_blk_mq_req_done()
2370 static bool mmc_blk_rw_wait_cond(struct mmc_queue *mq, int *err)  in mmc_blk_rw_wait_cond()  argument
2379 	spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_rw_wait_cond()
2380 	if (mq->recovery_needed) {  in mmc_blk_rw_wait_cond()
2384 		done = !mq->rw_wait;  in mmc_blk_rw_wait_cond()
2386 	mq->waiting = !done;  in mmc_blk_rw_wait_cond()
2387 	spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_rw_wait_cond()
2392 static int mmc_blk_rw_wait(struct mmc_queue *mq, struct request **prev_req)  in mmc_blk_rw_wait()  argument
2396 	wait_event(mq->wait, mmc_blk_rw_wait_cond(mq, &err));  in mmc_blk_rw_wait()
2399 	mmc_blk_mq_complete_prev_req(mq, prev_req);  in mmc_blk_rw_wait()
2404 static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq,  in mmc_blk_mq_issue_rw_rq()  argument
2408 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_issue_rw_rq()
2412 	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);  in mmc_blk_mq_issue_rw_rq()
2418 	err = mmc_blk_rw_wait(mq, &prev_req);  in mmc_blk_mq_issue_rw_rq()
2422 	mq->rw_wait = true;  in mmc_blk_mq_issue_rw_rq()
2427 		mmc_blk_mq_post_req(mq, prev_req, true);  in mmc_blk_mq_issue_rw_rq()
2430 		mq->rw_wait = false;  in mmc_blk_mq_issue_rw_rq()
2443 static int mmc_blk_wait_for_idle(struct mmc_queue *mq, struct mmc_host *host)  in mmc_blk_wait_for_idle()  argument
2448 	return mmc_blk_rw_wait(mq, NULL);  in mmc_blk_wait_for_idle()
2451 enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_issue_rq()  argument
2453 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_mq_issue_rq()
2462 	switch (mmc_issue_type(mq, req)) {  in mmc_blk_mq_issue_rq()
2464 		ret = mmc_blk_wait_for_idle(mq, host);  in mmc_blk_mq_issue_rq()
2470 			mmc_blk_issue_drv_op(mq, req);  in mmc_blk_mq_issue_rq()
2473 			mmc_blk_issue_discard_rq(mq, req);  in mmc_blk_mq_issue_rq()
2476 			mmc_blk_issue_secdiscard_rq(mq, req);  in mmc_blk_mq_issue_rq()
2479 			mmc_blk_issue_trim_rq(mq, req);  in mmc_blk_mq_issue_rq()
2482 			mmc_blk_issue_flush(mq, req);  in mmc_blk_mq_issue_rq()
2497 			ret = mmc_blk_cqe_issue_flush(mq, req);  in mmc_blk_mq_issue_rq()
2504 				ret = mmc_blk_cqe_issue_rw_rq(mq, req);  in mmc_blk_mq_issue_rq()
2506 				ret = mmc_blk_mq_issue_rw_rq(mq, req);  in mmc_blk_mq_issue_rq()
3130 	struct mmc_queue *mq = &md->queue;  in mmc_dbg_card_status_get()  local
3135 	req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0);  in mmc_dbg_card_status_get()
3160 	struct mmc_queue *mq = &md->queue;  in mmc_ext_csd_open()  local
3172 	req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0);  in mmc_ext_csd_open()