83 static void	mutex_init_robust(struct pthread *curthread);
146 mutex_assert_not_owned(struct pthread *curthread __unused,  in mutex_assert_not_owned()
157 	    (is_pshared_mutex(m) && curthread->robust_list ==  in mutex_assert_not_owned()
159 	    (!is_pshared_mutex(m) && curthread->priv_robust_list ==  in mutex_assert_not_owned()
164 		    m->m_rb_prev, (void *)curthread->robust_list,  in mutex_assert_not_owned()
165 		    (void *)curthread->priv_robust_list);  in mutex_assert_not_owned()
184 _mutex_enter_robust(struct pthread *curthread, struct pthread_mutex *m)  in _mutex_enter_robust()  argument
188 	if (__predict_false(curthread->inact_mtx != 0))  in _mutex_enter_robust()
194 	mutex_init_robust(curthread);  in _mutex_enter_robust()
195 	curthread->inact_mtx = (uintptr_t)&m->m_lock;  in _mutex_enter_robust()
200 _mutex_leave_robust(struct pthread *curthread, struct pthread_mutex *m __unused)  in _mutex_leave_robust()  argument
204 	if (__predict_false(curthread->inact_mtx != (uintptr_t)&m->m_lock))  in _mutex_leave_robust()
207 	curthread->inact_mtx = 0;  in _mutex_leave_robust()
224 mutex_init_robust(struct pthread *curthread)  in mutex_init_robust()  argument
228 	if (curthread == NULL)  in mutex_init_robust()
229 		curthread = _get_curthread();  in mutex_init_robust()
230 	if (curthread->robust_inited)  in mutex_init_robust()
232 	rb.robust_list_offset = (uintptr_t)&curthread->robust_list;  in mutex_init_robust()
233 	rb.robust_priv_list_offset = (uintptr_t)&curthread->priv_robust_list;  in mutex_init_robust()
234 	rb.robust_inact_offset = (uintptr_t)&curthread->inact_mtx;  in mutex_init_robust()
236 	curthread->robust_inited = 1;  in mutex_init_robust()
323 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m)  in set_inherited_priority()  argument
327 	m2 = TAILQ_LAST(&curthread->mq[mutex_qidx(m)], mutex_queue);  in set_inherited_priority()
439 queue_fork(struct pthread *curthread, struct mutex_queue *q,  in queue_fork()  argument
447 		m->m_lock.m_owner = TID(curthread) | bit;  in queue_fork()
452 _mutex_fork(struct pthread *curthread)  in _mutex_fork()  argument
455 	queue_fork(curthread, &curthread->mq[TMQ_NORM],  in _mutex_fork()
456 	    &curthread->mq[TMQ_NORM_PRIV], 0);  in _mutex_fork()
457 	queue_fork(curthread, &curthread->mq[TMQ_NORM_PP],  in _mutex_fork()
458 	    &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED);  in _mutex_fork()
459 	queue_fork(curthread, &curthread->mq[TMQ_ROBUST_PP],  in _mutex_fork()
460 	    &curthread->mq[TMQ_ROBUST_PP_PRIV], UMUTEX_CONTESTED);  in _mutex_fork()
461 	curthread->robust_list = 0;  in _mutex_fork()
525 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m,  in enqueue_mutex()  argument
534 		mutex_assert_not_owned(curthread, m);  in enqueue_mutex()
536 	TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);  in enqueue_mutex()
538 		TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe);  in enqueue_mutex()
540 		rl = is_pshared_mutex(m) ? &curthread->robust_list :  in enqueue_mutex()
541 		    &curthread->priv_robust_list;  in enqueue_mutex()
557 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)  in dequeue_mutex()  argument
568 				curthread->robust_list = m->m_lock.m_rb_lnk;  in dequeue_mutex()
570 				curthread->priv_robust_list =  in dequeue_mutex()
584 	TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe);  in dequeue_mutex()
586 		TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe);  in dequeue_mutex()
588 		set_inherited_priority(curthread, m);  in dequeue_mutex()
620 	struct pthread *curthread;  in __Tthr_mutex_trylock()  local
628 	curthread = _get_curthread();  in __Tthr_mutex_trylock()
629 	id = TID(curthread);  in __Tthr_mutex_trylock()
631 		THR_CRITICAL_ENTER(curthread);  in __Tthr_mutex_trylock()
632 	robust = _mutex_enter_robust(curthread, m);  in __Tthr_mutex_trylock()
635 		enqueue_mutex(curthread, m, ret);  in __Tthr_mutex_trylock()
642 		_mutex_leave_robust(curthread, m);  in __Tthr_mutex_trylock()
645 		THR_CRITICAL_LEAVE(curthread);  in __Tthr_mutex_trylock()
650 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,  in mutex_lock_sleep()  argument
656 	id = TID(curthread);  in mutex_lock_sleep()
710 		enqueue_mutex(curthread, m, ret);  in mutex_lock_sleep()
721 	struct pthread *curthread;  in mutex_lock_common()  local
725 	curthread  = _get_curthread();  in mutex_lock_common()
727 		THR_CRITICAL_ENTER(curthread);  in mutex_lock_common()
729 		robust = _mutex_enter_robust(curthread, m);  in mutex_lock_common()
730 	ret = _thr_umutex_trylock2(&m->m_lock, TID(curthread));  in mutex_lock_common()
732 		enqueue_mutex(curthread, m, ret);  in mutex_lock_common()
736 		ret = mutex_lock_sleep(curthread, m, abstime);  in mutex_lock_common()
739 		_mutex_leave_robust(curthread, m);  in mutex_lock_common()
742 		THR_CRITICAL_LEAVE(curthread);  in mutex_lock_common()
816 	struct pthread *curthread;  in _mutex_cv_attach()  local
818 	curthread = _get_curthread();  in _mutex_cv_attach()
819 	enqueue_mutex(curthread, m, 0);  in _mutex_cv_attach()
827 	struct pthread *curthread;  in _mutex_cv_detach()  local
830 	curthread = _get_curthread();  in _mutex_cv_detach()
831 	if ((error = _mutex_owned(curthread, mp)) != 0)  in _mutex_cv_detach()
839 	dequeue_mutex(curthread, mp);  in _mutex_cv_detach()
849 		_thr_wake_all(curthread->defer_waiters,  in _mutex_cv_detach()
850 		    curthread->nwaiter_defer);  in _mutex_cv_detach()
851 		curthread->nwaiter_defer = 0;  in _mutex_cv_detach()
957 	struct pthread *curthread;  in mutex_unlock_common()  local
967 	curthread = _get_curthread();  in mutex_unlock_common()
968 	id = TID(curthread);  in mutex_unlock_common()
988 		robust = _mutex_enter_robust(curthread, m);  in mutex_unlock_common()
989 		dequeue_mutex(curthread, m);  in mutex_unlock_common()
993 				_thr_wake_all(curthread->defer_waiters,  in mutex_unlock_common()
994 				    curthread->nwaiter_defer);  in mutex_unlock_common()
995 				curthread->nwaiter_defer = 0;  in mutex_unlock_common()
1000 			_mutex_leave_robust(curthread, m);  in mutex_unlock_common()
1003 		THR_CRITICAL_LEAVE(curthread);  in mutex_unlock_common()
1033 	struct pthread *curthread;  in _pthread_mutex_setprioceiling()  local
1055 	curthread = _get_curthread();  in _pthread_mutex_setprioceiling()
1056 	if (PMUTEX_OWNER_ID(m) == TID(curthread)) {  in _pthread_mutex_setprioceiling()
1063 			q = &curthread->mq[qidx];  in _pthread_mutex_setprioceiling()
1064 			qp = &curthread->mq[qidx + 1];  in _pthread_mutex_setprioceiling()
1163 _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp)  in _mutex_owned()  argument
1171 	if (PMUTEX_OWNER_ID(mp) != TID(curthread))  in _mutex_owned()
1180 	struct pthread *curthread;  in _Tthr_mutex_consistent()  local
1192 	curthread = _get_curthread();  in _Tthr_mutex_consistent()
1196 	if (PMUTEX_OWNER_ID(m) != TID(curthread))  in _Tthr_mutex_consistent()