118 	return &req->apoll->poll;  in io_poll_get_single()
131 static void io_init_poll_iocb(struct io_poll *poll, __poll_t events)  in io_init_poll_iocb()  argument
133 	poll->head = NULL;  in io_init_poll_iocb()
136 	poll->events = events | IO_POLL_UNMASK;  in io_init_poll_iocb()
137 	INIT_LIST_HEAD(&poll->wait.entry);  in io_init_poll_iocb()
138 	init_waitqueue_func_entry(&poll->wait, io_poll_wake);  in io_init_poll_iocb()
141 static inline void io_poll_remove_entry(struct io_poll *poll)  in io_poll_remove_entry()  argument
143 	struct wait_queue_head *head = smp_load_acquire(&poll->head);  in io_poll_remove_entry()
147 		list_del_init(&poll->wait.entry);  in io_poll_remove_entry()
148 		poll->head = NULL;  in io_poll_remove_entry()
334 			struct io_poll *poll;  in io_poll_task_func()  local
336 			poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_task_func()
337 			req->cqe.res = mangle_poll(req->cqe.res & poll->events);  in io_poll_task_func()
369 static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll)  in io_pollfree_wake()  argument
382 	list_del_init(&poll->wait.entry);  in io_pollfree_wake()
390 	smp_store_release(&poll->head, NULL);  in io_pollfree_wake()
398 	struct io_poll *poll = container_of(wait, struct io_poll, wait);  in io_poll_wake()  local
402 		return io_pollfree_wake(req, poll);  in io_poll_wake()
405 	if (mask && !(mask & (poll->events & ~IO_ASYNC_POLL_COMMON)))  in io_poll_wake()
415 			poll->events |= EPOLLONESHOT;  in io_poll_wake()
418 		if (mask && poll->events & EPOLLONESHOT) {  in io_poll_wake()
419 			list_del_init(&poll->wait.entry);  in io_poll_wake()
420 			poll->head = NULL;  in io_poll_wake()
435 	struct io_poll *poll = io_poll_get_single(req);  in io_poll_double_prepare()  local
439 	head = smp_load_acquire(&poll->head);  in io_poll_double_prepare()
457 static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt,  in __io_queue_proc()  argument
470 		struct io_poll *first = poll;  in __io_queue_proc()
483 		poll = kmalloc(sizeof(*poll), GFP_ATOMIC);  in __io_queue_proc()
484 		if (!poll) {  in __io_queue_proc()
491 		io_init_poll_iocb(poll, first->events);  in __io_queue_proc()
494 			kfree(poll);  in __io_queue_proc()
497 		*poll_ptr = poll;  in __io_queue_proc()
504 	poll->head = head;  in __io_queue_proc()
505 	poll->wait.private = (void *) wqe_private;  in __io_queue_proc()
507 	if (poll->events & EPOLLEXCLUSIVE) {  in __io_queue_proc()
508 		add_wait_queue_exclusive(head, &poll->wait);  in __io_queue_proc()
510 		add_wait_queue(head, &poll->wait);  in __io_queue_proc()
518 	struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll);  in io_poll_queue_proc()  local
520 	__io_queue_proc(poll, pt, head,  in io_poll_queue_proc()
546 				 struct io_poll *poll,  in __io_arm_poll_handler()  argument
551 	io_init_poll_iocb(poll, mask);  in __io_arm_poll_handler()
552 	poll->file = req->file;  in __io_arm_poll_handler()
553 	req->apoll_events = poll->events;  in __io_arm_poll_handler()
580 	if (poll->events & EPOLLEXCLUSIVE)  in __io_arm_poll_handler()
583 	mask = vfs_poll(req->file, &ipt->pt) & poll->events;  in __io_arm_poll_handler()
591 		} else if (mask && (poll->events & EPOLLET)) {  in __io_arm_poll_handler()
599 	   ((poll->events & (EPOLLET|EPOLLONESHOT)) == (EPOLLET|EPOLLONESHOT))) {  in __io_arm_poll_handler()
612 	if (mask && (poll->events & EPOLLET) &&  in __io_arm_poll_handler()
636 	__io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);  in io_async_queue_proc()
663 		apoll->poll.retries = APOLL_MAX_RETRY;  in io_req_alloc_apoll()
667 	if (unlikely(!--apoll->poll.retries))  in io_req_alloc_apoll()
708 	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags);  in io_arm_poll_handler()
711 	trace_io_uring_poll_arm(req, mask, apoll->poll.events);  in io_arm_poll_handler()
870 	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_add_prep()  local
881 	poll->events = io_poll_parse_events(sqe, flags);  in io_poll_add_prep()
887 	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_add()  local
893 	ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags);  in io_poll_add()
924 			struct io_poll *poll = io_kiocb_to_cmd(preq, struct io_poll);  in io_poll_remove()  local
926 			poll->events &= ~0xffff;  in io_poll_remove()
927 			poll->events |= poll_update->events & 0xffff;  in io_poll_remove()
928 			poll->events |= IO_POLL_UNMASK;  in io_poll_remove()