184 	return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);  in __io_cqring_events()
189 	return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head);  in __io_cqring_events_user()
385 	struct io_rings *r = ctx->rings;  in io_account_cq_overflow()
650 		atomic_andnot(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);  in __io_cqring_overflow_flush()
657 	if (ctx->rings)  in io_cqring_overflow_kill()
730 		atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);  in io_cqring_event_overflow()
759 	struct io_rings *rings = ctx->rings;  in io_cqe_cache_refill()  local
784 	ctx->cqe_cached = &rings->cqes[off];  in io_cqe_cache_refill()
1020 		atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);  in ctx_flush_and_put()
1193 			atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);  in io_req_local_work_add()
1218 		atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);  in io_req_normal_work_add()
1264 		atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);  in io_run_local_work_continue()
1299 		atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);  in __io_run_local_work()
2254 	struct io_rings *rings = ctx->rings;  in io_commit_sqring()  local
2261 	smp_store_release(&rings->sq.head, ctx->cached_sq_head);  in io_commit_sqring()
2285 			WRITE_ONCE(ctx->rings->sq_dropped,  in io_get_sqe()
2286 				   READ_ONCE(ctx->rings->sq_dropped) + 1);  in io_get_sqe()
2423 	if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail))  in io_cqring_min_timer_wakeup()
2536 	struct io_rings *rings = ctx->rings;  in io_cqring_wait()  local
2558 	iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;  in io_cqring_wait()
2559 	iowq.cq_min_tail = READ_ONCE(ctx->rings->cq.tail);  in io_cqring_wait()
2595 					READ_ONCE(ctx->rings->cq.tail);  in io_cqring_wait()
2654 	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;  in io_cqring_wait()
2661 	ctx->rings = NULL;  in io_rings_free()
2668 	struct io_rings *rings;  in rings_size()  local
2671 	off = struct_size(rings, cqes, cq_entries);  in rings_size()
3078 	if (!ctx->rings)  in io_uring_try_cancel_requests()
3481 	struct io_rings *rings;  in io_allocate_scq_urings()  local
3503 	ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);  in io_allocate_scq_urings()
3506 		ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);  in io_allocate_scq_urings()
3507 	rings->sq_ring_mask = p->sq_entries - 1;  in io_allocate_scq_urings()
3508 	rings->cq_ring_mask = p->cq_entries - 1;  in io_allocate_scq_urings()
3509 	rings->sq_ring_entries = p->sq_entries;  in io_allocate_scq_urings()
3510 	rings->cq_ring_entries = p->cq_entries;  in io_allocate_scq_urings()
3738 		p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;  in io_uring_create()