46 __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)  in __mutex_init()  argument
48 	atomic_long_set(&lock->owner, 0);  in __mutex_init()
49 	raw_spin_lock_init(&lock->wait_lock);  in __mutex_init()
50 	INIT_LIST_HEAD(&lock->wait_list);  in __mutex_init()
52 	osq_lock_init(&lock->osq);  in __mutex_init()
55 	debug_mutex_init(lock, name, key);  in __mutex_init()
64 bool mutex_is_locked(struct mutex *lock)  in mutex_is_locked()  argument
66 	return __mutex_owner(lock) != NULL;  in mutex_is_locked()
76 unsigned long mutex_get_owner(struct mutex *lock)  in mutex_get_owner()  argument
78 	unsigned long owner = atomic_long_read(&lock->owner);  in mutex_get_owner()
86 static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, bool handoff)  in __mutex_trylock_common()  argument
90 	owner = atomic_long_read(&lock->owner);  in __mutex_trylock_common()
112 		if (atomic_long_try_cmpxchg_acquire(&lock->owner, &owner, task | flags)) {  in __mutex_trylock_common()
125 static inline bool __mutex_trylock_or_handoff(struct mutex *lock, bool handoff)  in __mutex_trylock_or_handoff()  argument
127 	return !__mutex_trylock_common(lock, handoff);  in __mutex_trylock_or_handoff()
133 static inline bool __mutex_trylock(struct mutex *lock)  in __mutex_trylock()  argument
135 	return !__mutex_trylock_common(lock, false);  in __mutex_trylock()
149 static __always_inline bool __mutex_trylock_fast(struct mutex *lock)  in __mutex_trylock_fast()  argument
154 	MUTEX_WARN_ON(lock->magic != lock);  in __mutex_trylock_fast()
156 	if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr))  in __mutex_trylock_fast()
162 static __always_inline bool __mutex_unlock_fast(struct mutex *lock)  in __mutex_unlock_fast()  argument
166 	return atomic_long_try_cmpxchg_release(&lock->owner, &curr, 0UL);  in __mutex_unlock_fast()
170 static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag)  in __mutex_set_flag()  argument
172 	atomic_long_or(flag, &lock->owner);  in __mutex_set_flag()
175 static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag)  in __mutex_clear_flag()  argument
177 	atomic_long_andnot(flag, &lock->owner);  in __mutex_clear_flag()
180 static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter)  in __mutex_waiter_is_first()  argument
182 	return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;  in __mutex_waiter_is_first()
190 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,  in __mutex_add_waiter()  argument
194 	WRITE_ONCE(current->blocker_mutex, lock);  in __mutex_add_waiter()
196 	debug_mutex_add_waiter(lock, waiter, current);  in __mutex_add_waiter()
199 	if (__mutex_waiter_is_first(lock, waiter))  in __mutex_add_waiter()
200 		__mutex_set_flag(lock, MUTEX_FLAG_WAITERS);  in __mutex_add_waiter()
204 __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)  in __mutex_remove_waiter()  argument
207 	if (likely(list_empty(&lock->wait_list)))  in __mutex_remove_waiter()
208 		__mutex_clear_flag(lock, MUTEX_FLAGS);  in __mutex_remove_waiter()
210 	debug_mutex_remove_waiter(lock, waiter, current);  in __mutex_remove_waiter()
222 static void __mutex_handoff(struct mutex *lock, struct task_struct *task)  in __mutex_handoff()  argument
224 	unsigned long owner = atomic_long_read(&lock->owner);  in __mutex_handoff()
237 		if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, new))  in __mutex_handoff()
249 static void __sched __mutex_lock_slowpath(struct mutex *lock);
272 void __sched mutex_lock(struct mutex *lock)  in mutex_lock()  argument
276 	if (!__mutex_trylock_fast(lock))  in mutex_lock()
277 		__mutex_lock_slowpath(lock);  in mutex_lock()
289 static inline struct task_struct *__mutex_trylock_or_owner(struct mutex *lock)  in __mutex_trylock_or_owner()  argument
291 	return __mutex_trylock_common(lock, false);  in __mutex_trylock_or_owner()
295 bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in ww_mutex_spin_on_owner()  argument
300 	ww = container_of(lock, struct ww_mutex, base);  in ww_mutex_spin_on_owner()
323 	if (!waiter && (atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS))  in ww_mutex_spin_on_owner()
330 	if (waiter && !__mutex_waiter_is_first(lock, waiter))  in ww_mutex_spin_on_owner()
343 bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,  in mutex_spin_on_owner()  argument
350 	while (__mutex_owner(lock) == owner) {  in mutex_spin_on_owner()
369 		if (ww_ctx && !ww_mutex_spin_on_owner(lock, ww_ctx, waiter)) {  in mutex_spin_on_owner()
383 static inline int mutex_can_spin_on_owner(struct mutex *lock)  in mutex_can_spin_on_owner()  argument
398 	owner = __mutex_owner(lock);  in mutex_can_spin_on_owner()
432 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in mutex_optimistic_spin()  argument
443 		if (!mutex_can_spin_on_owner(lock))  in mutex_optimistic_spin()
451 		if (!osq_lock(&lock->osq))  in mutex_optimistic_spin()
459 		owner = __mutex_trylock_or_owner(lock);  in mutex_optimistic_spin()
467 		if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter))  in mutex_optimistic_spin()
480 		osq_unlock(&lock->osq);  in mutex_optimistic_spin()
487 		osq_unlock(&lock->osq);  in mutex_optimistic_spin()
508 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in mutex_optimistic_spin()  argument
515 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip);
533 void __sched mutex_unlock(struct mutex *lock)  in mutex_unlock()  argument
536 	if (__mutex_unlock_fast(lock))  in mutex_unlock()
539 	__mutex_unlock_slowpath(lock, _RET_IP_);  in mutex_unlock()
554 void __sched ww_mutex_unlock(struct ww_mutex *lock)  in ww_mutex_unlock()  argument
556 	__ww_mutex_unlock(lock);  in ww_mutex_unlock()
557 	mutex_unlock(&lock->base);  in ww_mutex_unlock()
565 __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass,  in __mutex_lock_common()  argument
580 	MUTEX_WARN_ON(lock->magic != lock);  in __mutex_lock_common()
582 	ww = container_of(lock, struct ww_mutex, base);  in __mutex_lock_common()
601 	mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);  in __mutex_lock_common()
603 	trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);  in __mutex_lock_common()
604 	if (__mutex_trylock(lock) ||  in __mutex_lock_common()
605 	    mutex_optimistic_spin(lock, ww_ctx, NULL)) {  in __mutex_lock_common()
607 		lock_acquired(&lock->dep_map, ip);  in __mutex_lock_common()
610 		trace_contention_end(lock, 0);  in __mutex_lock_common()
615 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in __mutex_lock_common()
619 	if (__mutex_trylock(lock)) {  in __mutex_lock_common()
621 			__ww_mutex_check_waiters(lock, ww_ctx, &wake_q);  in __mutex_lock_common()
626 	debug_mutex_lock_common(lock, &waiter);  in __mutex_lock_common()
631 	lock_contended(&lock->dep_map, ip);  in __mutex_lock_common()
635 		__mutex_add_waiter(lock, &waiter, &lock->wait_list);  in __mutex_lock_common()
641 		ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx, &wake_q);  in __mutex_lock_common()
647 	trace_contention_begin(lock, LCB_F_MUTEX);  in __mutex_lock_common()
657 		if (__mutex_trylock(lock))  in __mutex_lock_common()
671 			ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx);  in __mutex_lock_common()
676 		raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);  in __mutex_lock_common()
680 		first = __mutex_waiter_is_first(lock, &waiter);  in __mutex_lock_common()
688 		if (__mutex_trylock_or_handoff(lock, first))  in __mutex_lock_common()
692 			trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);  in __mutex_lock_common()
693 			if (mutex_optimistic_spin(lock, ww_ctx, &waiter))  in __mutex_lock_common()
695 			trace_contention_begin(lock, LCB_F_MUTEX);  in __mutex_lock_common()
698 		raw_spin_lock_irqsave(&lock->wait_lock, flags);  in __mutex_lock_common()
700 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in __mutex_lock_common()
710 		    !__mutex_waiter_is_first(lock, &waiter))  in __mutex_lock_common()
711 			__ww_mutex_check_waiters(lock, ww_ctx, &wake_q);  in __mutex_lock_common()
714 	__mutex_remove_waiter(lock, &waiter);  in __mutex_lock_common()
720 	lock_acquired(&lock->dep_map, ip);  in __mutex_lock_common()
721 	trace_contention_end(lock, 0);  in __mutex_lock_common()
726 	raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);  in __mutex_lock_common()
732 	__mutex_remove_waiter(lock, &waiter);  in __mutex_lock_common()
734 	trace_contention_end(lock, ret);  in __mutex_lock_common()
735 	raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);  in __mutex_lock_common()
737 	mutex_release(&lock->dep_map, ip);  in __mutex_lock_common()
743 __mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,  in __mutex_lock()  argument
746 	return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, false);  in __mutex_lock()
750 __ww_mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,  in __ww_mutex_lock()  argument
753 	return __mutex_lock_common(lock, state, subclass, NULL, ip, ww_ctx, true);  in __ww_mutex_lock()
796 mutex_lock_nested(struct mutex *lock, unsigned int subclass)  in mutex_lock_nested()  argument
798 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);  in mutex_lock_nested()
804 _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)  in _mutex_lock_nest_lock()  argument
806 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_);  in _mutex_lock_nest_lock()
811 mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)  in mutex_lock_killable_nested()  argument
813 	return __mutex_lock(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_);  in mutex_lock_killable_nested()
818 mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)  in mutex_lock_interruptible_nested()  argument
820 	return __mutex_lock(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_);  in mutex_lock_interruptible_nested()
825 mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)  in mutex_lock_io_nested()  argument
832 	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,  in mutex_lock_io_nested()
839 ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in ww_mutex_deadlock_injection()  argument
853 		ctx->contending_lock = lock;  in ww_mutex_deadlock_injection()
855 		ww_mutex_unlock(lock);  in ww_mutex_deadlock_injection()
865 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in ww_mutex_lock()  argument
870 	ret =  __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE,  in ww_mutex_lock()
873 		return ww_mutex_deadlock_injection(lock, ctx);  in ww_mutex_lock()
880 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in ww_mutex_lock_interruptible()  argument
885 	ret = __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE,  in ww_mutex_lock_interruptible()
889 		return ww_mutex_deadlock_injection(lock, ctx);  in ww_mutex_lock_interruptible()
900 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)  in __mutex_unlock_slowpath()  argument
907 	mutex_release(&lock->dep_map, ip);  in __mutex_unlock_slowpath()
916 	owner = atomic_long_read(&lock->owner);  in __mutex_unlock_slowpath()
924 		if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) {  in __mutex_unlock_slowpath()
932 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in __mutex_unlock_slowpath()
933 	debug_mutex_unlock(lock);  in __mutex_unlock_slowpath()
934 	if (!list_empty(&lock->wait_list)) {  in __mutex_unlock_slowpath()
937 			list_first_entry(&lock->wait_list,  in __mutex_unlock_slowpath()
942 		debug_mutex_wake_waiter(lock, waiter);  in __mutex_unlock_slowpath()
947 		__mutex_handoff(lock, next);  in __mutex_unlock_slowpath()
949 	raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);  in __mutex_unlock_slowpath()
958 __mutex_lock_killable_slowpath(struct mutex *lock);
961 __mutex_lock_interruptible_slowpath(struct mutex *lock);
975 int __sched mutex_lock_interruptible(struct mutex *lock)  in mutex_lock_interruptible()  argument
979 	if (__mutex_trylock_fast(lock))  in mutex_lock_interruptible()
982 	return __mutex_lock_interruptible_slowpath(lock);  in mutex_lock_interruptible()
999 int __sched mutex_lock_killable(struct mutex *lock)  in mutex_lock_killable()  argument
1003 	if (__mutex_trylock_fast(lock))  in mutex_lock_killable()
1006 	return __mutex_lock_killable_slowpath(lock);  in mutex_lock_killable()
1020 void __sched mutex_lock_io(struct mutex *lock)  in mutex_lock_io()  argument
1025 	mutex_lock(lock);  in mutex_lock_io()
1031 __mutex_lock_slowpath(struct mutex *lock)  in __mutex_lock_slowpath()  argument
1033 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);  in __mutex_lock_slowpath()
1037 __mutex_lock_killable_slowpath(struct mutex *lock)  in __mutex_lock_killable_slowpath()  argument
1039 	return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);  in __mutex_lock_killable_slowpath()
1043 __mutex_lock_interruptible_slowpath(struct mutex *lock)  in __mutex_lock_interruptible_slowpath()  argument
1045 	return __mutex_lock(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);  in __mutex_lock_interruptible_slowpath()
1049 __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in __ww_mutex_lock_slowpath()  argument
1051 	return __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 0,  in __ww_mutex_lock_slowpath()
1056 __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,  in __ww_mutex_lock_interruptible_slowpath()  argument
1059 	return __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 0,  in __ww_mutex_lock_interruptible_slowpath()
1079 int __sched mutex_trylock(struct mutex *lock)  in mutex_trylock()  argument
1083 	MUTEX_WARN_ON(lock->magic != lock);  in mutex_trylock()
1085 	locked = __mutex_trylock(lock);  in mutex_trylock()
1087 		mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);  in mutex_trylock()
1095 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in ww_mutex_lock()  argument
1099 	if (__mutex_trylock_fast(&lock->base)) {  in ww_mutex_lock()
1101 			ww_mutex_set_context_fastpath(lock, ctx);  in ww_mutex_lock()
1105 	return __ww_mutex_lock_slowpath(lock, ctx);  in ww_mutex_lock()
1110 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in ww_mutex_lock_interruptible()  argument
1114 	if (__mutex_trylock_fast(&lock->base)) {  in ww_mutex_lock_interruptible()
1116 			ww_mutex_set_context_fastpath(lock, ctx);  in ww_mutex_lock_interruptible()
1120 	return __ww_mutex_lock_interruptible_slowpath(lock, ctx);  in ww_mutex_lock_interruptible()
1137 int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)  in atomic_dec_and_mutex_lock()  argument
1143 	mutex_lock(lock);  in atomic_dec_and_mutex_lock()
1146 		mutex_unlock(lock);  in atomic_dec_and_mutex_lock()