787 	struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work);  in rb_wake_up_waiters()  local
790 	(void)atomic_fetch_inc_release(&rbwork->seq);  in rb_wake_up_waiters()
792 	wake_up_all(&rbwork->waiters);  in rb_wake_up_waiters()
793 	if (rbwork->full_waiters_pending || rbwork->wakeup_full) {  in rb_wake_up_waiters()
796 			container_of(rbwork, struct ring_buffer_per_cpu, irq_work);  in rb_wake_up_waiters()
800 		rbwork->wakeup_full = false;  in rb_wake_up_waiters()
801 		rbwork->full_waiters_pending = false;  in rb_wake_up_waiters()
807 		wake_up_all(&rbwork->full_waiters);  in rb_wake_up_waiters()
822 	struct rb_irq_work *rbwork;  in ring_buffer_wake_waiters()  local
833 		rbwork = &buffer->irq_work;  in ring_buffer_wake_waiters()
844 		rbwork = &cpu_buffer->irq_work;  in ring_buffer_wake_waiters()
848 	irq_work_queue(&rbwork->work);  in ring_buffer_wake_waiters()
883 rb_wait_cond(struct rb_irq_work *rbwork, struct trace_buffer *buffer,  in rb_wait_cond()  argument
913 		rbwork->full_waiters_pending = true;  in rb_wait_cond()
915 		rbwork->waiters_pending = true;  in rb_wait_cond()
932 	struct rb_irq_work *rbwork = rdata->irq_work;  in rb_wait_once()  local
934 	return atomic_read_acquire(&rbwork->seq) != rdata->seq;  in rb_wait_once()
954 	struct rb_irq_work *rbwork;  in ring_buffer_wait()  local
964 		rbwork = &buffer->irq_work;  in ring_buffer_wait()
971 		rbwork = &cpu_buffer->irq_work;  in ring_buffer_wait()
975 		waitq = &rbwork->full_waiters;  in ring_buffer_wait()
977 		waitq = &rbwork->waiters;  in ring_buffer_wait()
982 		rdata.irq_work = rbwork;  in ring_buffer_wait()
983 		rdata.seq = atomic_read_acquire(&rbwork->seq);  in ring_buffer_wait()
988 				rb_wait_cond(rbwork, buffer, cpu, full, cond, data));  in ring_buffer_wait()
1012 	struct rb_irq_work *rbwork;  in ring_buffer_poll_wait()  local
1015 		rbwork = &buffer->irq_work;  in ring_buffer_poll_wait()
1022 		rbwork = &cpu_buffer->irq_work;  in ring_buffer_poll_wait()
1026 		poll_wait(filp, &rbwork->full_waiters, poll_table);  in ring_buffer_poll_wait()
1043 		rbwork->full_waiters_pending = true;  in ring_buffer_poll_wait()
1047 	poll_wait(filp, &rbwork->waiters, poll_table);  in ring_buffer_poll_wait()
1048 	rbwork->waiters_pending = true;  in ring_buffer_poll_wait()