36 static inline int __ww_mutex_add_waiter(struct rt_mutex_waiter *waiter,  in __ww_mutex_add_waiter()  argument
56 					struct rt_mutex_waiter *waiter,  in __ww_mutex_check_kill()  argument
363 waiter_update_prio(struct rt_mutex_waiter *waiter, struct task_struct *task)  in waiter_update_prio()  argument
365 	lockdep_assert_held(&waiter->lock->wait_lock);  in waiter_update_prio()
366 	lockdep_assert(RB_EMPTY_NODE(&waiter->tree.entry));  in waiter_update_prio()
368 	waiter->tree.prio = __waiter_prio(task);  in waiter_update_prio()
369 	waiter->tree.deadline = task->dl.deadline;  in waiter_update_prio()
376 waiter_clone_prio(struct rt_mutex_waiter *waiter, struct task_struct *task)  in waiter_clone_prio()  argument
378 	lockdep_assert_held(&waiter->lock->wait_lock);  in waiter_clone_prio()
380 	lockdep_assert(RB_EMPTY_NODE(&waiter->pi_tree.entry));  in waiter_clone_prio()
382 	waiter->pi_tree.prio = waiter->tree.prio;  in waiter_clone_prio()
383 	waiter->pi_tree.deadline = waiter->tree.deadline;  in waiter_clone_prio()
430 static inline bool rt_mutex_steal(struct rt_mutex_waiter *waiter,  in rt_mutex_steal()  argument
433 	if (rt_waiter_node_less(&waiter->tree, &top_waiter->tree))  in rt_mutex_steal()
441 	if (rt_or_dl_prio(waiter->tree.prio))  in rt_mutex_steal()
444 	return rt_waiter_node_equal(&waiter->tree, &top_waiter->tree);  in rt_mutex_steal()
480 rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue()  argument
484 	rb_add_cached(&waiter->tree.entry, &lock->waiters, __waiter_less);  in rt_mutex_enqueue()
488 rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue()  argument
492 	if (RB_EMPTY_NODE(&waiter->tree.entry))  in rt_mutex_dequeue()
495 	rb_erase_cached(&waiter->tree.entry, &lock->waiters);  in rt_mutex_dequeue()
496 	RB_CLEAR_NODE(&waiter->tree.entry);  in rt_mutex_dequeue()
508 rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue_pi()  argument
512 	rb_add_cached(&waiter->pi_tree.entry, &task->pi_waiters, __pi_waiter_less);  in rt_mutex_enqueue_pi()
516 rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue_pi()  argument
520 	if (RB_EMPTY_NODE(&waiter->pi_tree.entry))  in rt_mutex_dequeue_pi()
523 	rb_erase_cached(&waiter->pi_tree.entry, &task->pi_waiters);  in rt_mutex_dequeue_pi()
524 	RB_CLEAR_NODE(&waiter->pi_tree.entry);  in rt_mutex_dequeue_pi()
592 rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,  in rt_mutex_cond_detect_deadlock()  argument
596 		return waiter != NULL;  in rt_mutex_cond_detect_deadlock()
685 	struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;  in rt_mutex_adjust_prio_chain()  local
737 	waiter = task->pi_blocked_on;  in rt_mutex_adjust_prio_chain()
748 	if (!waiter)  in rt_mutex_adjust_prio_chain()
767 	if (next_lock != waiter->lock)  in rt_mutex_adjust_prio_chain()
792 	if (IS_ENABLED(CONFIG_PREEMPT_RT) && waiter->ww_ctx && detect_deadlock)  in rt_mutex_adjust_prio_chain()
824 	if (rt_waiter_node_equal(&waiter->tree, task_to_waiter_node(task))) {  in rt_mutex_adjust_prio_chain()
835 	lock = waiter->lock;  in rt_mutex_adjust_prio_chain()
935 	rt_mutex_dequeue(lock, waiter);  in rt_mutex_adjust_prio_chain()
948 	waiter_update_prio(waiter, task);  in rt_mutex_adjust_prio_chain()
950 	rt_mutex_enqueue(lock, waiter);  in rt_mutex_adjust_prio_chain()
992 	if (waiter == rt_mutex_top_waiter(lock)) {  in rt_mutex_adjust_prio_chain()
1000 		waiter_clone_prio(waiter, task);  in rt_mutex_adjust_prio_chain()
1001 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
1004 	} else if (prerequeue_top_waiter == waiter) {  in rt_mutex_adjust_prio_chain()
1015 		rt_mutex_dequeue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
1016 		waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
1017 		waiter_clone_prio(waiter, task);  in rt_mutex_adjust_prio_chain()
1018 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
1063 	if (!detect_deadlock && waiter != top_waiter)  in rt_mutex_adjust_prio_chain()
1088 		     struct rt_mutex_waiter *waiter)  in try_to_take_rt_mutex()  argument
1122 	if (waiter) {  in try_to_take_rt_mutex()
1129 		if (waiter == top_waiter || rt_mutex_steal(waiter, top_waiter)) {  in try_to_take_rt_mutex()
1134 			rt_mutex_dequeue(lock, waiter);  in try_to_take_rt_mutex()
1204 					   struct rt_mutex_waiter *waiter,  in task_blocks_on_rt_mutex()  argument
1211 	struct rt_mutex_waiter *top_waiter = waiter;  in task_blocks_on_rt_mutex()
1233 	waiter->task = task;  in task_blocks_on_rt_mutex()
1234 	waiter->lock = lock;  in task_blocks_on_rt_mutex()
1235 	waiter_update_prio(waiter, task);  in task_blocks_on_rt_mutex()
1236 	waiter_clone_prio(waiter, task);  in task_blocks_on_rt_mutex()
1241 	rt_mutex_enqueue(lock, waiter);  in task_blocks_on_rt_mutex()
1243 	task->pi_blocked_on = waiter;  in task_blocks_on_rt_mutex()
1252 		res = __ww_mutex_add_waiter(waiter, rtm, ww_ctx, wake_q);  in task_blocks_on_rt_mutex()
1255 			rt_mutex_dequeue(lock, waiter);  in task_blocks_on_rt_mutex()
1266 	if (waiter == rt_mutex_top_waiter(lock)) {  in task_blocks_on_rt_mutex()
1268 		rt_mutex_enqueue_pi(owner, waiter);  in task_blocks_on_rt_mutex()
1273 	} else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {  in task_blocks_on_rt_mutex()
1299 					 next_lock, waiter, task);  in task_blocks_on_rt_mutex()
1315 	struct rt_mutex_waiter *waiter;  in mark_wakeup_next_waiter()  local
1321 	waiter = rt_mutex_top_waiter(lock);  in mark_wakeup_next_waiter()
1330 	rt_mutex_dequeue_pi(current, waiter);  in mark_wakeup_next_waiter()
1354 	rt_mutex_wake_q_add(wqh, waiter);  in mark_wakeup_next_waiter()
1482 				  struct rt_mutex_waiter *waiter,  in rtmutex_spin_on_owner()  argument
1508 		    !rt_mutex_waiter_is_top_waiter(lock, waiter)) {  in rtmutex_spin_on_owner()
1519 				  struct rt_mutex_waiter *waiter,  in rtmutex_spin_on_owner()  argument
1540 				  struct rt_mutex_waiter *waiter)  in remove_waiter()  argument
1542 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));  in remove_waiter()
1549 	rt_mutex_dequeue(lock, waiter);  in remove_waiter()
1562 	rt_mutex_dequeue_pi(owner, waiter);  in remove_waiter()
1608 					   struct rt_mutex_waiter *waiter,  in rt_mutex_slowlock_block()  argument
1619 		if (try_to_take_rt_mutex(lock, current, waiter)) {  in rt_mutex_slowlock_block()
1634 			ret = __ww_mutex_check_kill(rtm, waiter, ww_ctx);  in rt_mutex_slowlock_block()
1639 		if (waiter == rt_mutex_top_waiter(lock))  in rt_mutex_slowlock_block()
1645 		if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner)) {  in rt_mutex_slowlock_block()
1695 				       struct rt_mutex_waiter *waiter,  in __rt_mutex_slowlock()  argument
1719 	ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk, wake_q);  in __rt_mutex_slowlock()
1721 		ret = rt_mutex_slowlock_block(lock, ww_ctx, state, NULL, waiter, wake_q);  in __rt_mutex_slowlock()
1733 		remove_waiter(lock, waiter);  in __rt_mutex_slowlock()
1734 		rt_mutex_handle_deadlock(ret, chwalk, lock, waiter);  in __rt_mutex_slowlock()
1754 	struct rt_mutex_waiter waiter;  in __rt_mutex_slowlock_locked()  local
1757 	rt_mutex_init_waiter(&waiter);  in __rt_mutex_slowlock_locked()
1758 	waiter.ww_ctx = ww_ctx;  in __rt_mutex_slowlock_locked()
1761 				  &waiter, wake_q);  in __rt_mutex_slowlock_locked()
1763 	debug_rt_mutex_free_waiter(&waiter);  in __rt_mutex_slowlock_locked()
1833 	struct rt_mutex_waiter waiter;  in rtlock_slowlock_locked()  local
1844 	rt_mutex_init_rtlock_waiter(&waiter);  in rtlock_slowlock_locked()
1851 	task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK, wake_q);  in rtlock_slowlock_locked()
1855 		if (try_to_take_rt_mutex(lock, current, &waiter)) {  in rtlock_slowlock_locked()
1860 		if (&waiter == rt_mutex_top_waiter(lock))  in rtlock_slowlock_locked()
1866 		if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner)) {  in rtlock_slowlock_locked()
1883 	debug_rt_mutex_free_waiter(&waiter);  in rtlock_slowlock_locked()