83 static void	mutex_init_robust(struct pthread *curthread);
146 mutex_assert_not_owned(struct pthread *curthread __unused,  in mutex_assert_not_owned()
157 	    (is_pshared_mutex(m) && curthread->robust_list ==  in mutex_assert_not_owned()
159 	    (!is_pshared_mutex(m) && curthread->priv_robust_list ==  in mutex_assert_not_owned()
164 		    m->m_rb_prev, (void *)curthread->robust_list,  in mutex_assert_not_owned()
165 		    (void *)curthread->priv_robust_list);  in mutex_assert_not_owned()
184 _mutex_enter_robust(struct pthread *curthread, struct pthread_mutex *m)  in _mutex_enter_robust()  argument
188 	if (__predict_false(curthread->inact_mtx != 0))  in _mutex_enter_robust()
194 	mutex_init_robust(curthread);  in _mutex_enter_robust()
195 	curthread->inact_mtx = (uintptr_t)&m->m_lock;  in _mutex_enter_robust()
200 _mutex_leave_robust(struct pthread *curthread, struct pthread_mutex *m __unused)  in _mutex_leave_robust()  argument
204 	if (__predict_false(curthread->inact_mtx != (uintptr_t)&m->m_lock))  in _mutex_leave_robust()
207 	curthread->inact_mtx = 0;  in _mutex_leave_robust()
224 mutex_init_robust(struct pthread *curthread)  in mutex_init_robust()  argument
228 	if (curthread == NULL)  in mutex_init_robust()
229 		curthread = _get_curthread();  in mutex_init_robust()
230 	if (curthread->robust_inited)  in mutex_init_robust()
232 	rb.robust_list_offset = (uintptr_t)&curthread->robust_list;  in mutex_init_robust()
233 	rb.robust_priv_list_offset = (uintptr_t)&curthread->priv_robust_list;  in mutex_init_robust()
234 	rb.robust_inact_offset = (uintptr_t)&curthread->inact_mtx;  in mutex_init_robust()
236 	curthread->robust_inited = 1;  in mutex_init_robust()
323 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m)  in set_inherited_priority()  argument
327 	m2 = TAILQ_LAST(&curthread->mq[mutex_qidx(m)], mutex_queue);  in set_inherited_priority()
437 queue_fork(struct pthread *curthread, struct mutex_queue *q,  in queue_fork()  argument
445 		m->m_lock.m_owner = TID(curthread) | bit;  in queue_fork()
450 _mutex_fork(struct pthread *curthread)  in _mutex_fork()  argument
453 	queue_fork(curthread, &curthread->mq[TMQ_NORM],  in _mutex_fork()
454 	    &curthread->mq[TMQ_NORM_PRIV], 0);  in _mutex_fork()
455 	queue_fork(curthread, &curthread->mq[TMQ_NORM_PP],  in _mutex_fork()
456 	    &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED);  in _mutex_fork()
457 	queue_fork(curthread, &curthread->mq[TMQ_ROBUST_PP],  in _mutex_fork()
458 	    &curthread->mq[TMQ_ROBUST_PP_PRIV], UMUTEX_CONTESTED);  in _mutex_fork()
459 	curthread->robust_list = 0;  in _mutex_fork()
523 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m,  in enqueue_mutex()  argument
532 		mutex_assert_not_owned(curthread, m);  in enqueue_mutex()
534 	TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);  in enqueue_mutex()
536 		TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe);  in enqueue_mutex()
538 		rl = is_pshared_mutex(m) ? &curthread->robust_list :  in enqueue_mutex()
539 		    &curthread->priv_robust_list;  in enqueue_mutex()
555 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)  in dequeue_mutex()  argument
566 				curthread->robust_list = m->m_lock.m_rb_lnk;  in dequeue_mutex()
568 				curthread->priv_robust_list =  in dequeue_mutex()
582 	TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe);  in dequeue_mutex()
584 		TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe);  in dequeue_mutex()
586 		set_inherited_priority(curthread, m);  in dequeue_mutex()
618 	struct pthread *curthread;  in __Tthr_mutex_trylock()  local
626 	curthread = _get_curthread();  in __Tthr_mutex_trylock()
627 	id = TID(curthread);  in __Tthr_mutex_trylock()
629 		THR_CRITICAL_ENTER(curthread);  in __Tthr_mutex_trylock()
630 	robust = _mutex_enter_robust(curthread, m);  in __Tthr_mutex_trylock()
633 		enqueue_mutex(curthread, m, ret);  in __Tthr_mutex_trylock()
640 		_mutex_leave_robust(curthread, m);  in __Tthr_mutex_trylock()
643 		THR_CRITICAL_LEAVE(curthread);  in __Tthr_mutex_trylock()
648 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,  in mutex_lock_sleep()  argument
654 	id = TID(curthread);  in mutex_lock_sleep()
708 		enqueue_mutex(curthread, m, ret);  in mutex_lock_sleep()
719 	struct pthread *curthread;  in mutex_lock_common()  local
723 	curthread  = _get_curthread();  in mutex_lock_common()
725 		THR_CRITICAL_ENTER(curthread);  in mutex_lock_common()
727 		robust = _mutex_enter_robust(curthread, m);  in mutex_lock_common()
728 	ret = _thr_umutex_trylock2(&m->m_lock, TID(curthread));  in mutex_lock_common()
730 		enqueue_mutex(curthread, m, ret);  in mutex_lock_common()
734 		ret = mutex_lock_sleep(curthread, m, abstime);  in mutex_lock_common()
737 		_mutex_leave_robust(curthread, m);  in mutex_lock_common()
740 		THR_CRITICAL_LEAVE(curthread);  in mutex_lock_common()
814 	struct pthread *curthread;  in _mutex_cv_attach()  local
816 	curthread = _get_curthread();  in _mutex_cv_attach()
817 	enqueue_mutex(curthread, m, 0);  in _mutex_cv_attach()
825 	struct pthread *curthread;  in _mutex_cv_detach()  local
828 	curthread = _get_curthread();  in _mutex_cv_detach()
829 	if ((error = _mutex_owned(curthread, mp)) != 0)  in _mutex_cv_detach()
837 	dequeue_mutex(curthread, mp);  in _mutex_cv_detach()
847 		_thr_wake_all(curthread->defer_waiters,  in _mutex_cv_detach()
848 		    curthread->nwaiter_defer);  in _mutex_cv_detach()
849 		curthread->nwaiter_defer = 0;  in _mutex_cv_detach()
955 	struct pthread *curthread;  in mutex_unlock_common()  local
965 	curthread = _get_curthread();  in mutex_unlock_common()
966 	id = TID(curthread);  in mutex_unlock_common()
986 		robust = _mutex_enter_robust(curthread, m);  in mutex_unlock_common()
987 		dequeue_mutex(curthread, m);  in mutex_unlock_common()
991 				_thr_wake_all(curthread->defer_waiters,  in mutex_unlock_common()
992 				    curthread->nwaiter_defer);  in mutex_unlock_common()
993 				curthread->nwaiter_defer = 0;  in mutex_unlock_common()
998 			_mutex_leave_robust(curthread, m);  in mutex_unlock_common()
1001 		THR_CRITICAL_LEAVE(curthread);  in mutex_unlock_common()
1031 	struct pthread *curthread;  in _pthread_mutex_setprioceiling()  local
1053 	curthread = _get_curthread();  in _pthread_mutex_setprioceiling()
1054 	if (PMUTEX_OWNER_ID(m) == TID(curthread)) {  in _pthread_mutex_setprioceiling()
1061 			q = &curthread->mq[qidx];  in _pthread_mutex_setprioceiling()
1062 			qp = &curthread->mq[qidx + 1];  in _pthread_mutex_setprioceiling()
1161 _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp)  in _mutex_owned()  argument
1169 	if (PMUTEX_OWNER_ID(mp) != TID(curthread))  in _mutex_owned()
1178 	struct pthread *curthread;  in _Tthr_mutex_consistent()  local
1190 	curthread = _get_curthread();  in _Tthr_mutex_consistent()
1194 	if (PMUTEX_OWNER_ID(m) != TID(curthread))  in _Tthr_mutex_consistent()