15 	struct futex_pi_state *pi_state;  in refill_pi_state_cache()  local
20 	pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL);  in refill_pi_state_cache()
22 	if (!pi_state)  in refill_pi_state_cache()
25 	INIT_LIST_HEAD(&pi_state->list);  in refill_pi_state_cache()
27 	pi_state->owner = NULL;  in refill_pi_state_cache()
28 	refcount_set(&pi_state->refcount, 1);  in refill_pi_state_cache()
29 	pi_state->key = FUTEX_KEY_INIT;  in refill_pi_state_cache()
31 	current->pi_state_cache = pi_state;  in refill_pi_state_cache()
38 	struct futex_pi_state *pi_state = current->pi_state_cache;  in alloc_pi_state()  local
40 	WARN_ON(!pi_state);  in alloc_pi_state()
43 	return pi_state;  in alloc_pi_state()
46 static void pi_state_update_owner(struct futex_pi_state *pi_state,  in pi_state_update_owner()  argument
49 	struct task_struct *old_owner = pi_state->owner;  in pi_state_update_owner()
51 	lockdep_assert_held(&pi_state->pi_mutex.wait_lock);  in pi_state_update_owner()
55 		WARN_ON(list_empty(&pi_state->list));  in pi_state_update_owner()
56 		list_del_init(&pi_state->list);  in pi_state_update_owner()
62 		WARN_ON(!list_empty(&pi_state->list));  in pi_state_update_owner()
63 		list_add(&pi_state->list, &new_owner->pi_state_list);  in pi_state_update_owner()
64 		pi_state->owner = new_owner;  in pi_state_update_owner()
69 void get_pi_state(struct futex_pi_state *pi_state)  in get_pi_state()  argument
71 	WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount));  in get_pi_state()
78 void put_pi_state(struct futex_pi_state *pi_state)  in put_pi_state()  argument
80 	if (!pi_state)  in put_pi_state()
83 	if (!refcount_dec_and_test(&pi_state->refcount))  in put_pi_state()
90 	if (pi_state->owner) {  in put_pi_state()
93 		raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags);  in put_pi_state()
94 		pi_state_update_owner(pi_state, NULL);  in put_pi_state()
95 		rt_mutex_proxy_unlock(&pi_state->pi_mutex);  in put_pi_state()
96 		raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags);  in put_pi_state()
100 		kfree(pi_state);  in put_pi_state()
107 		pi_state->owner = NULL;  in put_pi_state()
108 		refcount_set(&pi_state->refcount, 1);  in put_pi_state()
109 		current->pi_state_cache = pi_state;  in put_pi_state()
204 			      struct futex_pi_state *pi_state,  in attach_to_pi_state()  argument
214 	if (unlikely(!pi_state))  in attach_to_pi_state()
229 	WARN_ON(!refcount_read(&pi_state->refcount));  in attach_to_pi_state()
235 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
258 		if (!pi_state->owner) {  in attach_to_pi_state()
286 		if (!pi_state->owner)  in attach_to_pi_state()
295 	if (pid != task_pid_vnr(pi_state->owner))  in attach_to_pi_state()
299 	get_pi_state(pi_state);  in attach_to_pi_state()
300 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
301 	*ps = pi_state;  in attach_to_pi_state()
317 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in attach_to_pi_state()
386 	struct futex_pi_state *pi_state = alloc_pi_state();  in __attach_to_pi_owner()  local
392 	rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);  in __attach_to_pi_owner()
395 	pi_state->key = *key;  in __attach_to_pi_owner()
397 	WARN_ON(!list_empty(&pi_state->list));  in __attach_to_pi_owner()
398 	list_add(&pi_state->list, &p->pi_state_list);  in __attach_to_pi_owner()
403 	pi_state->owner = p;  in __attach_to_pi_owner()
405 	*ps = pi_state;  in __attach_to_pi_owner()
551 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);  in futex_lock_pi_atomic()
615 			 struct futex_pi_state *pi_state,  in wake_futex_pi()  argument
658 		pi_state_update_owner(pi_state, new_owner);  in wake_futex_pi()
659 		postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wqh);  in wake_futex_pi()
663 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in wake_futex_pi()
674 	struct futex_pi_state *pi_state = q->pi_state;  in __fixup_pi_state_owner()  local
679 	oldowner = pi_state->owner;  in __fixup_pi_state_owner()
714 		if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) {  in __fixup_pi_state_owner()
723 		newowner = rt_mutex_owner(&pi_state->pi_mutex);  in __fixup_pi_state_owner()
750 	if (!pi_state->owner)  in __fixup_pi_state_owner()
773 	pi_state_update_owner(pi_state, newowner);  in __fixup_pi_state_owner()
791 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in __fixup_pi_state_owner()
810 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in __fixup_pi_state_owner()
815 	if (pi_state->owner != oldowner)  in __fixup_pi_state_owner()
838 	pi_state_update_owner(pi_state, rt_mutex_owner(&pi_state->pi_mutex));  in __fixup_pi_state_owner()
846 	struct futex_pi_state *pi_state = q->pi_state;  in fixup_pi_state_owner()  local
851 	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
853 	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in fixup_pi_state_owner()
883 		if (q->pi_state->owner != current)  in fixup_pi_owner()
896 	if (q->pi_state->owner == current)  in fixup_pi_owner()
903 	if (WARN_ON_ONCE(rt_mutex_owner(&q->pi_state->pi_mutex) == current))  in fixup_pi_owner()
944 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,  in futex_lock_pi()
980 	WARN_ON(!q.pi_state);  in futex_lock_pi()
988 		ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);  in futex_lock_pi()
1015 	raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);  in futex_lock_pi()
1022 	ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current, &wake_q);  in futex_lock_pi()
1023 	raw_spin_unlock_irq_wake(&q.pi_state->pi_mutex.wait_lock, &wake_q);  in futex_lock_pi()
1034 	ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);  in futex_lock_pi()
1055 	if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))  in futex_lock_pi()
1148 		struct futex_pi_state *pi_state = top_waiter->pi_state;  in futex_unlock_pi()  local
1152 		if (!pi_state)  in futex_unlock_pi()
1159 		if (pi_state->owner != current)  in futex_unlock_pi()
1177 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);  in futex_unlock_pi()
1188 		rt_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);  in futex_unlock_pi()
1191 			raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);  in futex_unlock_pi()
1195 		get_pi_state(pi_state);  in futex_unlock_pi()
1199 		ret = wake_futex_pi(uaddr, uval, pi_state, rt_waiter);  in futex_unlock_pi()
1201 		put_pi_state(pi_state);  in futex_unlock_pi()