80 static void	mutex_init_robust(struct pthread *curthread);
143 mutex_assert_not_owned(struct pthread *curthread __unused,  in mutex_assert_not_owned()
154 	    (is_pshared_mutex(m) && curthread->robust_list ==  in mutex_assert_not_owned()
156 	    (!is_pshared_mutex(m) && curthread->priv_robust_list ==  in mutex_assert_not_owned()
161 		    m->m_rb_prev, (void *)curthread->robust_list,  in mutex_assert_not_owned()
162 		    (void *)curthread->priv_robust_list);  in mutex_assert_not_owned()
181 _mutex_enter_robust(struct pthread *curthread, struct pthread_mutex *m)  in _mutex_enter_robust()  argument
185 	if (__predict_false(curthread->inact_mtx != 0))  in _mutex_enter_robust()
191 	mutex_init_robust(curthread);  in _mutex_enter_robust()
192 	curthread->inact_mtx = (uintptr_t)&m->m_lock;  in _mutex_enter_robust()
197 _mutex_leave_robust(struct pthread *curthread, struct pthread_mutex *m __unused)  in _mutex_leave_robust()  argument
201 	if (__predict_false(curthread->inact_mtx != (uintptr_t)&m->m_lock))  in _mutex_leave_robust()
204 	curthread->inact_mtx = 0;  in _mutex_leave_robust()
221 mutex_init_robust(struct pthread *curthread)  in mutex_init_robust()  argument
225 	if (curthread == NULL)  in mutex_init_robust()
226 		curthread = _get_curthread();  in mutex_init_robust()
227 	if (curthread->robust_inited)  in mutex_init_robust()
229 	rb.robust_list_offset = (uintptr_t)&curthread->robust_list;  in mutex_init_robust()
230 	rb.robust_priv_list_offset = (uintptr_t)&curthread->priv_robust_list;  in mutex_init_robust()
231 	rb.robust_inact_offset = (uintptr_t)&curthread->inact_mtx;  in mutex_init_robust()
233 	curthread->robust_inited = 1;  in mutex_init_robust()
320 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m)  in set_inherited_priority()  argument
324 	m2 = TAILQ_LAST(&curthread->mq[mutex_qidx(m)], mutex_queue);  in set_inherited_priority()
436 queue_fork(struct pthread *curthread, struct mutex_queue *q,  in queue_fork()  argument
444 		m->m_lock.m_owner = TID(curthread) | bit;  in queue_fork()
449 _mutex_fork(struct pthread *curthread)  in _mutex_fork()  argument
452 	queue_fork(curthread, &curthread->mq[TMQ_NORM],  in _mutex_fork()
453 	    &curthread->mq[TMQ_NORM_PRIV], 0);  in _mutex_fork()
454 	queue_fork(curthread, &curthread->mq[TMQ_NORM_PP],  in _mutex_fork()
455 	    &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED);  in _mutex_fork()
456 	queue_fork(curthread, &curthread->mq[TMQ_ROBUST_PP],  in _mutex_fork()
457 	    &curthread->mq[TMQ_ROBUST_PP_PRIV], UMUTEX_CONTESTED);  in _mutex_fork()
458 	curthread->robust_list = 0;  in _mutex_fork()
522 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m,  in enqueue_mutex()  argument
531 		mutex_assert_not_owned(curthread, m);  in enqueue_mutex()
533 	TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);  in enqueue_mutex()
535 		TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe);  in enqueue_mutex()
537 		rl = is_pshared_mutex(m) ? &curthread->robust_list :  in enqueue_mutex()
538 		    &curthread->priv_robust_list;  in enqueue_mutex()
554 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)  in dequeue_mutex()  argument
565 				curthread->robust_list = m->m_lock.m_rb_lnk;  in dequeue_mutex()
567 				curthread->priv_robust_list =  in dequeue_mutex()
581 	TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe);  in dequeue_mutex()
583 		TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe);  in dequeue_mutex()
585 		set_inherited_priority(curthread, m);  in dequeue_mutex()
617 	struct pthread *curthread;  in __Tthr_mutex_trylock()  local
625 	curthread = _get_curthread();  in __Tthr_mutex_trylock()
626 	id = TID(curthread);  in __Tthr_mutex_trylock()
628 		THR_CRITICAL_ENTER(curthread);  in __Tthr_mutex_trylock()
629 	robust = _mutex_enter_robust(curthread, m);  in __Tthr_mutex_trylock()
632 		enqueue_mutex(curthread, m, ret);  in __Tthr_mutex_trylock()
639 		_mutex_leave_robust(curthread, m);  in __Tthr_mutex_trylock()
642 		THR_CRITICAL_LEAVE(curthread);  in __Tthr_mutex_trylock()
647 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,  in mutex_lock_sleep()  argument
653 	id = TID(curthread);  in mutex_lock_sleep()
707 		enqueue_mutex(curthread, m, ret);  in mutex_lock_sleep()
718 	struct pthread *curthread;  in mutex_lock_common()  local
722 	curthread  = _get_curthread();  in mutex_lock_common()
724 		THR_CRITICAL_ENTER(curthread);  in mutex_lock_common()
726 		robust = _mutex_enter_robust(curthread, m);  in mutex_lock_common()
727 	ret = _thr_umutex_trylock2(&m->m_lock, TID(curthread));  in mutex_lock_common()
729 		enqueue_mutex(curthread, m, ret);  in mutex_lock_common()
733 		ret = mutex_lock_sleep(curthread, m, abstime);  in mutex_lock_common()
736 		_mutex_leave_robust(curthread, m);  in mutex_lock_common()
739 		THR_CRITICAL_LEAVE(curthread);  in mutex_lock_common()
813 	struct pthread *curthread;  in _mutex_cv_attach()  local
815 	curthread = _get_curthread();  in _mutex_cv_attach()
816 	enqueue_mutex(curthread, m, 0);  in _mutex_cv_attach()
824 	struct pthread *curthread;  in _mutex_cv_detach()  local
827 	curthread = _get_curthread();  in _mutex_cv_detach()
828 	if ((error = _mutex_owned(curthread, mp)) != 0)  in _mutex_cv_detach()
836 	dequeue_mutex(curthread, mp);  in _mutex_cv_detach()
846 		_thr_wake_all(curthread->defer_waiters,  in _mutex_cv_detach()
847 		    curthread->nwaiter_defer);  in _mutex_cv_detach()
848 		curthread->nwaiter_defer = 0;  in _mutex_cv_detach()
954 	struct pthread *curthread;  in mutex_unlock_common()  local
964 	curthread = _get_curthread();  in mutex_unlock_common()
965 	id = TID(curthread);  in mutex_unlock_common()
985 		robust = _mutex_enter_robust(curthread, m);  in mutex_unlock_common()
986 		dequeue_mutex(curthread, m);  in mutex_unlock_common()
990 				_thr_wake_all(curthread->defer_waiters,  in mutex_unlock_common()
991 				    curthread->nwaiter_defer);  in mutex_unlock_common()
992 				curthread->nwaiter_defer = 0;  in mutex_unlock_common()
997 			_mutex_leave_robust(curthread, m);  in mutex_unlock_common()
1000 		THR_CRITICAL_LEAVE(curthread);  in mutex_unlock_common()
1030 	struct pthread *curthread;  in _pthread_mutex_setprioceiling()  local
1052 	curthread = _get_curthread();  in _pthread_mutex_setprioceiling()
1053 	if (PMUTEX_OWNER_ID(m) == TID(curthread)) {  in _pthread_mutex_setprioceiling()
1060 			q = &curthread->mq[qidx];  in _pthread_mutex_setprioceiling()
1061 			qp = &curthread->mq[qidx + 1];  in _pthread_mutex_setprioceiling()
1160 _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp)  in _mutex_owned()  argument
1168 	if (PMUTEX_OWNER_ID(mp) != TID(curthread))  in _mutex_owned()
1177 	struct pthread *curthread;  in _Tthr_mutex_consistent()  local
1189 	curthread = _get_curthread();  in _Tthr_mutex_consistent()
1193 	if (PMUTEX_OWNER_ID(m) != TID(curthread))  in _Tthr_mutex_consistent()