Lines Matching refs:lck
70 static kmp_int32 __kmp_get_tas_lock_owner(kmp_tas_lock_t *lck) { in __kmp_get_tas_lock_owner() argument
71 return KMP_LOCK_STRIP(KMP_ATOMIC_LD_RLX(&lck->lk.poll)) - 1; in __kmp_get_tas_lock_owner()
74 static inline bool __kmp_is_tas_lock_nestable(kmp_tas_lock_t *lck) { in __kmp_is_tas_lock_nestable() argument
75 return lck->lk.depth_locked != -1; in __kmp_is_tas_lock_nestable()
79 __kmp_acquire_tas_lock_timed_template(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_tas_lock_timed_template() argument
83 kmp_uint32 curr = KMP_LOCK_STRIP(lck->lk.poll); in __kmp_acquire_tas_lock_timed_template()
85 __kmp_printf("LOCK CONTENTION: %p\n", lck); in __kmp_acquire_tas_lock_timed_template()
92 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free && in __kmp_acquire_tas_lock_timed_template()
93 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) { in __kmp_acquire_tas_lock_timed_template()
94 KMP_FSYNC_ACQUIRED(lck); in __kmp_acquire_tas_lock_timed_template()
100 KMP_FSYNC_PREPARE(lck); in __kmp_acquire_tas_lock_timed_template()
112 } while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != tas_free || in __kmp_acquire_tas_lock_timed_template()
113 !__kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)); in __kmp_acquire_tas_lock_timed_template()
114 KMP_FSYNC_ACQUIRED(lck); in __kmp_acquire_tas_lock_timed_template()
118 int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_tas_lock() argument
119 int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid); in __kmp_acquire_tas_lock()
123 static int __kmp_acquire_tas_lock_with_checks(kmp_tas_lock_t *lck, in __kmp_acquire_tas_lock_with_checks() argument
127 __kmp_is_tas_lock_nestable(lck)) { in __kmp_acquire_tas_lock_with_checks()
130 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) == gtid)) { in __kmp_acquire_tas_lock_with_checks()
133 return __kmp_acquire_tas_lock(lck, gtid); in __kmp_acquire_tas_lock_with_checks()
136 int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_test_tas_lock() argument
139 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free && in __kmp_test_tas_lock()
140 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) { in __kmp_test_tas_lock()
141 KMP_FSYNC_ACQUIRED(lck); in __kmp_test_tas_lock()
147 static int __kmp_test_tas_lock_with_checks(kmp_tas_lock_t *lck, in __kmp_test_tas_lock_with_checks() argument
151 __kmp_is_tas_lock_nestable(lck)) { in __kmp_test_tas_lock_with_checks()
154 return __kmp_test_tas_lock(lck, gtid); in __kmp_test_tas_lock_with_checks()
157 int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_release_tas_lock() argument
160 KMP_FSYNC_RELEASING(lck); in __kmp_release_tas_lock()
161 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(tas)); in __kmp_release_tas_lock()
168 static int __kmp_release_tas_lock_with_checks(kmp_tas_lock_t *lck, in __kmp_release_tas_lock_with_checks() argument
173 __kmp_is_tas_lock_nestable(lck)) { in __kmp_release_tas_lock_with_checks()
176 if (__kmp_get_tas_lock_owner(lck) == -1) { in __kmp_release_tas_lock_with_checks()
179 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) >= 0) && in __kmp_release_tas_lock_with_checks()
180 (__kmp_get_tas_lock_owner(lck) != gtid)) { in __kmp_release_tas_lock_with_checks()
183 return __kmp_release_tas_lock(lck, gtid); in __kmp_release_tas_lock_with_checks()
186 void __kmp_init_tas_lock(kmp_tas_lock_t *lck) { in __kmp_init_tas_lock() argument
187 lck->lk.poll = KMP_LOCK_FREE(tas); in __kmp_init_tas_lock()
190 void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck) { lck->lk.poll = 0; } in __kmp_destroy_tas_lock() argument
192 static void __kmp_destroy_tas_lock_with_checks(kmp_tas_lock_t *lck) { in __kmp_destroy_tas_lock_with_checks() argument
195 __kmp_is_tas_lock_nestable(lck)) { in __kmp_destroy_tas_lock_with_checks()
198 if (__kmp_get_tas_lock_owner(lck) != -1) { in __kmp_destroy_tas_lock_with_checks()
201 __kmp_destroy_tas_lock(lck); in __kmp_destroy_tas_lock_with_checks()
206 int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_nested_tas_lock() argument
209 if (__kmp_get_tas_lock_owner(lck) == gtid) { in __kmp_acquire_nested_tas_lock()
210 lck->lk.depth_locked += 1; in __kmp_acquire_nested_tas_lock()
213 __kmp_acquire_tas_lock_timed_template(lck, gtid); in __kmp_acquire_nested_tas_lock()
214 lck->lk.depth_locked = 1; in __kmp_acquire_nested_tas_lock()
219 static int __kmp_acquire_nested_tas_lock_with_checks(kmp_tas_lock_t *lck, in __kmp_acquire_nested_tas_lock_with_checks() argument
222 if (!__kmp_is_tas_lock_nestable(lck)) { in __kmp_acquire_nested_tas_lock_with_checks()
225 return __kmp_acquire_nested_tas_lock(lck, gtid); in __kmp_acquire_nested_tas_lock_with_checks()
228 int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_test_nested_tas_lock() argument
233 if (__kmp_get_tas_lock_owner(lck) == gtid) { in __kmp_test_nested_tas_lock()
234 retval = ++lck->lk.depth_locked; in __kmp_test_nested_tas_lock()
235 } else if (!__kmp_test_tas_lock(lck, gtid)) { in __kmp_test_nested_tas_lock()
239 retval = lck->lk.depth_locked = 1; in __kmp_test_nested_tas_lock()
244 static int __kmp_test_nested_tas_lock_with_checks(kmp_tas_lock_t *lck, in __kmp_test_nested_tas_lock_with_checks() argument
247 if (!__kmp_is_tas_lock_nestable(lck)) { in __kmp_test_nested_tas_lock_with_checks()
250 return __kmp_test_nested_tas_lock(lck, gtid); in __kmp_test_nested_tas_lock_with_checks()
253 int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_release_nested_tas_lock() argument
257 if (--(lck->lk.depth_locked) == 0) { in __kmp_release_nested_tas_lock()
258 __kmp_release_tas_lock(lck, gtid); in __kmp_release_nested_tas_lock()
264 static int __kmp_release_nested_tas_lock_with_checks(kmp_tas_lock_t *lck, in __kmp_release_nested_tas_lock_with_checks() argument
268 if (!__kmp_is_tas_lock_nestable(lck)) { in __kmp_release_nested_tas_lock_with_checks()
271 if (__kmp_get_tas_lock_owner(lck) == -1) { in __kmp_release_nested_tas_lock_with_checks()
274 if (__kmp_get_tas_lock_owner(lck) != gtid) { in __kmp_release_nested_tas_lock_with_checks()
277 return __kmp_release_nested_tas_lock(lck, gtid); in __kmp_release_nested_tas_lock_with_checks()
280 void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck) { in __kmp_init_nested_tas_lock() argument
281 __kmp_init_tas_lock(lck); in __kmp_init_nested_tas_lock()
282 lck->lk.depth_locked = 0; // >= 0 for nestable locks, -1 for simple locks in __kmp_init_nested_tas_lock()
285 void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck) { in __kmp_destroy_nested_tas_lock() argument
286 __kmp_destroy_tas_lock(lck); in __kmp_destroy_nested_tas_lock()
287 lck->lk.depth_locked = 0; in __kmp_destroy_nested_tas_lock()
290 static void __kmp_destroy_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) { in __kmp_destroy_nested_tas_lock_with_checks() argument
292 if (!__kmp_is_tas_lock_nestable(lck)) { in __kmp_destroy_nested_tas_lock_with_checks()
295 if (__kmp_get_tas_lock_owner(lck) != -1) { in __kmp_destroy_nested_tas_lock_with_checks()
298 __kmp_destroy_nested_tas_lock(lck); in __kmp_destroy_nested_tas_lock_with_checks()
311 static kmp_int32 __kmp_get_futex_lock_owner(kmp_futex_lock_t *lck) { in __kmp_get_futex_lock_owner() argument
312 return KMP_LOCK_STRIP((TCR_4(lck->lk.poll) >> 1)) - 1; in __kmp_get_futex_lock_owner()
315 static inline bool __kmp_is_futex_lock_nestable(kmp_futex_lock_t *lck) { in __kmp_is_futex_lock_nestable() argument
316 return lck->lk.depth_locked != -1; in __kmp_is_futex_lock_nestable()
320 __kmp_acquire_futex_lock_timed_template(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_futex_lock_timed_template() argument
326 kmp_uint32 curr = KMP_LOCK_STRIP(TCR_4(lck->lk.poll)); in __kmp_acquire_futex_lock_timed_template()
328 __kmp_printf("LOCK CONTENTION: %p\n", lck); in __kmp_acquire_futex_lock_timed_template()
332 KMP_FSYNC_PREPARE(lck); in __kmp_acquire_futex_lock_timed_template()
334 lck, lck->lk.poll, gtid)); in __kmp_acquire_futex_lock_timed_template()
339 &(lck->lk.poll), KMP_LOCK_FREE(futex), in __kmp_acquire_futex_lock_timed_template()
346 lck, gtid, poll_val, cond)); in __kmp_acquire_futex_lock_timed_template()
357 if (!KMP_COMPARE_AND_STORE_REL32(&(lck->lk.poll), poll_val, in __kmp_acquire_futex_lock_timed_template()
362 lck, lck->lk.poll, gtid)); in __kmp_acquire_futex_lock_timed_template()
368 ("__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n", lck, in __kmp_acquire_futex_lock_timed_template()
369 lck->lk.poll, gtid)); in __kmp_acquire_futex_lock_timed_template()
375 lck, gtid, poll_val)); in __kmp_acquire_futex_lock_timed_template()
378 if ((rc = syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAIT, poll_val, NULL, in __kmp_acquire_futex_lock_timed_template()
382 lck, gtid, poll_val, rc, errno)); in __kmp_acquire_futex_lock_timed_template()
388 lck, gtid, poll_val)); in __kmp_acquire_futex_lock_timed_template()
395 KMP_FSYNC_ACQUIRED(lck); in __kmp_acquire_futex_lock_timed_template()
396 KA_TRACE(1000, ("__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck, in __kmp_acquire_futex_lock_timed_template()
397 lck->lk.poll, gtid)); in __kmp_acquire_futex_lock_timed_template()
401 int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_futex_lock() argument
402 int retval = __kmp_acquire_futex_lock_timed_template(lck, gtid); in __kmp_acquire_futex_lock()
406 static int __kmp_acquire_futex_lock_with_checks(kmp_futex_lock_t *lck, in __kmp_acquire_futex_lock_with_checks() argument
410 __kmp_is_futex_lock_nestable(lck)) { in __kmp_acquire_futex_lock_with_checks()
413 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) == gtid)) { in __kmp_acquire_futex_lock_with_checks()
416 return __kmp_acquire_futex_lock(lck, gtid); in __kmp_acquire_futex_lock_with_checks()
419 int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_test_futex_lock() argument
420 if (KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(futex), in __kmp_test_futex_lock()
422 KMP_FSYNC_ACQUIRED(lck); in __kmp_test_futex_lock()
428 static int __kmp_test_futex_lock_with_checks(kmp_futex_lock_t *lck, in __kmp_test_futex_lock_with_checks() argument
432 __kmp_is_futex_lock_nestable(lck)) { in __kmp_test_futex_lock_with_checks()
435 return __kmp_test_futex_lock(lck, gtid); in __kmp_test_futex_lock_with_checks()
438 int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_release_futex_lock() argument
442 lck, lck->lk.poll, gtid)); in __kmp_release_futex_lock()
444 KMP_FSYNC_RELEASING(lck); in __kmp_release_futex_lock()
446 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(lck->lk.poll), KMP_LOCK_FREE(futex)); in __kmp_release_futex_lock()
450 lck, gtid, poll_val)); in __kmp_release_futex_lock()
455 lck, gtid)); in __kmp_release_futex_lock()
456 syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex), in __kmp_release_futex_lock()
462 KA_TRACE(1000, ("__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck, in __kmp_release_futex_lock()
463 lck->lk.poll, gtid)); in __kmp_release_futex_lock()
469 static int __kmp_release_futex_lock_with_checks(kmp_futex_lock_t *lck, in __kmp_release_futex_lock_with_checks() argument
474 __kmp_is_futex_lock_nestable(lck)) { in __kmp_release_futex_lock_with_checks()
477 if (__kmp_get_futex_lock_owner(lck) == -1) { in __kmp_release_futex_lock_with_checks()
480 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) >= 0) && in __kmp_release_futex_lock_with_checks()
481 (__kmp_get_futex_lock_owner(lck) != gtid)) { in __kmp_release_futex_lock_with_checks()
484 return __kmp_release_futex_lock(lck, gtid); in __kmp_release_futex_lock_with_checks()
487 void __kmp_init_futex_lock(kmp_futex_lock_t *lck) { in __kmp_init_futex_lock() argument
488 TCW_4(lck->lk.poll, KMP_LOCK_FREE(futex)); in __kmp_init_futex_lock()
491 void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck) { lck->lk.poll = 0; } in __kmp_destroy_futex_lock() argument
493 static void __kmp_destroy_futex_lock_with_checks(kmp_futex_lock_t *lck) { in __kmp_destroy_futex_lock_with_checks() argument
496 __kmp_is_futex_lock_nestable(lck)) { in __kmp_destroy_futex_lock_with_checks()
499 if (__kmp_get_futex_lock_owner(lck) != -1) { in __kmp_destroy_futex_lock_with_checks()
502 __kmp_destroy_futex_lock(lck); in __kmp_destroy_futex_lock_with_checks()
507 int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_nested_futex_lock() argument
510 if (__kmp_get_futex_lock_owner(lck) == gtid) { in __kmp_acquire_nested_futex_lock()
511 lck->lk.depth_locked += 1; in __kmp_acquire_nested_futex_lock()
514 __kmp_acquire_futex_lock_timed_template(lck, gtid); in __kmp_acquire_nested_futex_lock()
515 lck->lk.depth_locked = 1; in __kmp_acquire_nested_futex_lock()
520 static int __kmp_acquire_nested_futex_lock_with_checks(kmp_futex_lock_t *lck, in __kmp_acquire_nested_futex_lock_with_checks() argument
523 if (!__kmp_is_futex_lock_nestable(lck)) { in __kmp_acquire_nested_futex_lock_with_checks()
526 return __kmp_acquire_nested_futex_lock(lck, gtid); in __kmp_acquire_nested_futex_lock_with_checks()
529 int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_test_nested_futex_lock() argument
534 if (__kmp_get_futex_lock_owner(lck) == gtid) { in __kmp_test_nested_futex_lock()
535 retval = ++lck->lk.depth_locked; in __kmp_test_nested_futex_lock()
536 } else if (!__kmp_test_futex_lock(lck, gtid)) { in __kmp_test_nested_futex_lock()
540 retval = lck->lk.depth_locked = 1; in __kmp_test_nested_futex_lock()
545 static int __kmp_test_nested_futex_lock_with_checks(kmp_futex_lock_t *lck, in __kmp_test_nested_futex_lock_with_checks() argument
548 if (!__kmp_is_futex_lock_nestable(lck)) { in __kmp_test_nested_futex_lock_with_checks()
551 return __kmp_test_nested_futex_lock(lck, gtid); in __kmp_test_nested_futex_lock_with_checks()
554 int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_release_nested_futex_lock() argument
558 if (--(lck->lk.depth_locked) == 0) { in __kmp_release_nested_futex_lock()
559 __kmp_release_futex_lock(lck, gtid); in __kmp_release_nested_futex_lock()
565 static int __kmp_release_nested_futex_lock_with_checks(kmp_futex_lock_t *lck, in __kmp_release_nested_futex_lock_with_checks() argument
569 if (!__kmp_is_futex_lock_nestable(lck)) { in __kmp_release_nested_futex_lock_with_checks()
572 if (__kmp_get_futex_lock_owner(lck) == -1) { in __kmp_release_nested_futex_lock_with_checks()
575 if (__kmp_get_futex_lock_owner(lck) != gtid) { in __kmp_release_nested_futex_lock_with_checks()
578 return __kmp_release_nested_futex_lock(lck, gtid); in __kmp_release_nested_futex_lock_with_checks()
581 void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck) { in __kmp_init_nested_futex_lock() argument
582 __kmp_init_futex_lock(lck); in __kmp_init_nested_futex_lock()
583 lck->lk.depth_locked = 0; // >= 0 for nestable locks, -1 for simple locks in __kmp_init_nested_futex_lock()
586 void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck) { in __kmp_destroy_nested_futex_lock() argument
587 __kmp_destroy_futex_lock(lck); in __kmp_destroy_nested_futex_lock()
588 lck->lk.depth_locked = 0; in __kmp_destroy_nested_futex_lock()
591 static void __kmp_destroy_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) { in __kmp_destroy_nested_futex_lock_with_checks() argument
593 if (!__kmp_is_futex_lock_nestable(lck)) { in __kmp_destroy_nested_futex_lock_with_checks()
596 if (__kmp_get_futex_lock_owner(lck) != -1) { in __kmp_destroy_nested_futex_lock_with_checks()
599 __kmp_destroy_nested_futex_lock(lck); in __kmp_destroy_nested_futex_lock_with_checks()
607 static kmp_int32 __kmp_get_ticket_lock_owner(kmp_ticket_lock_t *lck) { in __kmp_get_ticket_lock_owner() argument
608 return std::atomic_load_explicit(&lck->lk.owner_id, in __kmp_get_ticket_lock_owner()
613 static inline bool __kmp_is_ticket_lock_nestable(kmp_ticket_lock_t *lck) { in __kmp_is_ticket_lock_nestable() argument
614 return std::atomic_load_explicit(&lck->lk.depth_locked, in __kmp_is_ticket_lock_nestable()
624 __kmp_acquire_ticket_lock_timed_template(kmp_ticket_lock_t *lck, in __kmp_acquire_ticket_lock_timed_template() argument
627 &lck->lk.next_ticket, 1U, std::memory_order_relaxed); in __kmp_acquire_ticket_lock_timed_template()
630 if (std::atomic_load_explicit(&lck->lk.now_serving, in __kmp_acquire_ticket_lock_timed_template()
632 __kmp_printf("LOCK CONTENTION: %p\n", lck); in __kmp_acquire_ticket_lock_timed_template()
636 if (std::atomic_load_explicit(&lck->lk.now_serving, in __kmp_acquire_ticket_lock_timed_template()
640 KMP_WAIT_PTR(&lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck); in __kmp_acquire_ticket_lock_timed_template()
644 int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_ticket_lock() argument
645 int retval = __kmp_acquire_ticket_lock_timed_template(lck, gtid); in __kmp_acquire_ticket_lock()
649 static int __kmp_acquire_ticket_lock_with_checks(kmp_ticket_lock_t *lck, in __kmp_acquire_ticket_lock_with_checks() argument
653 if (!std::atomic_load_explicit(&lck->lk.initialized, in __kmp_acquire_ticket_lock_with_checks()
657 if (lck->lk.self != lck) { in __kmp_acquire_ticket_lock_with_checks()
660 if (__kmp_is_ticket_lock_nestable(lck)) { in __kmp_acquire_ticket_lock_with_checks()
663 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) == gtid)) { in __kmp_acquire_ticket_lock_with_checks()
667 __kmp_acquire_ticket_lock(lck, gtid); in __kmp_acquire_ticket_lock_with_checks()
669 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1, in __kmp_acquire_ticket_lock_with_checks()
674 int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_test_ticket_lock() argument
675 kmp_uint32 my_ticket = std::atomic_load_explicit(&lck->lk.next_ticket, in __kmp_test_ticket_lock()
678 if (std::atomic_load_explicit(&lck->lk.now_serving, in __kmp_test_ticket_lock()
682 &lck->lk.next_ticket, &my_ticket, next_ticket, in __kmp_test_ticket_lock()
690 static int __kmp_test_ticket_lock_with_checks(kmp_ticket_lock_t *lck, in __kmp_test_ticket_lock_with_checks() argument
694 if (!std::atomic_load_explicit(&lck->lk.initialized, in __kmp_test_ticket_lock_with_checks()
698 if (lck->lk.self != lck) { in __kmp_test_ticket_lock_with_checks()
701 if (__kmp_is_ticket_lock_nestable(lck)) { in __kmp_test_ticket_lock_with_checks()
705 int retval = __kmp_test_ticket_lock(lck, gtid); in __kmp_test_ticket_lock_with_checks()
708 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1, in __kmp_test_ticket_lock_with_checks()
714 int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_release_ticket_lock() argument
715 kmp_uint32 distance = std::atomic_load_explicit(&lck->lk.next_ticket, in __kmp_release_ticket_lock()
717 std::atomic_load_explicit(&lck->lk.now_serving, in __kmp_release_ticket_lock()
720 std::atomic_fetch_add_explicit(&lck->lk.now_serving, 1U, in __kmp_release_ticket_lock()
728 static int __kmp_release_ticket_lock_with_checks(kmp_ticket_lock_t *lck, in __kmp_release_ticket_lock_with_checks() argument
732 if (!std::atomic_load_explicit(&lck->lk.initialized, in __kmp_release_ticket_lock_with_checks()
736 if (lck->lk.self != lck) { in __kmp_release_ticket_lock_with_checks()
739 if (__kmp_is_ticket_lock_nestable(lck)) { in __kmp_release_ticket_lock_with_checks()
742 if (__kmp_get_ticket_lock_owner(lck) == -1) { in __kmp_release_ticket_lock_with_checks()
745 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) >= 0) && in __kmp_release_ticket_lock_with_checks()
746 (__kmp_get_ticket_lock_owner(lck) != gtid)) { in __kmp_release_ticket_lock_with_checks()
749 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed); in __kmp_release_ticket_lock_with_checks()
750 return __kmp_release_ticket_lock(lck, gtid); in __kmp_release_ticket_lock_with_checks()
753 void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck) { in __kmp_init_ticket_lock() argument
754 lck->lk.location = NULL; in __kmp_init_ticket_lock()
755 lck->lk.self = lck; in __kmp_init_ticket_lock()
756 std::atomic_store_explicit(&lck->lk.next_ticket, 0U, in __kmp_init_ticket_lock()
758 std::atomic_store_explicit(&lck->lk.now_serving, 0U, in __kmp_init_ticket_lock()
761 &lck->lk.owner_id, 0, in __kmp_init_ticket_lock()
764 &lck->lk.depth_locked, -1, in __kmp_init_ticket_lock()
766 std::atomic_store_explicit(&lck->lk.initialized, true, in __kmp_init_ticket_lock()
770 void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck) { in __kmp_destroy_ticket_lock() argument
771 std::atomic_store_explicit(&lck->lk.initialized, false, in __kmp_destroy_ticket_lock()
773 lck->lk.self = NULL; in __kmp_destroy_ticket_lock()
774 lck->lk.location = NULL; in __kmp_destroy_ticket_lock()
775 std::atomic_store_explicit(&lck->lk.next_ticket, 0U, in __kmp_destroy_ticket_lock()
777 std::atomic_store_explicit(&lck->lk.now_serving, 0U, in __kmp_destroy_ticket_lock()
779 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed); in __kmp_destroy_ticket_lock()
780 std::atomic_store_explicit(&lck->lk.depth_locked, -1, in __kmp_destroy_ticket_lock()
784 static void __kmp_destroy_ticket_lock_with_checks(kmp_ticket_lock_t *lck) { in __kmp_destroy_ticket_lock_with_checks() argument
787 if (!std::atomic_load_explicit(&lck->lk.initialized, in __kmp_destroy_ticket_lock_with_checks()
791 if (lck->lk.self != lck) { in __kmp_destroy_ticket_lock_with_checks()
794 if (__kmp_is_ticket_lock_nestable(lck)) { in __kmp_destroy_ticket_lock_with_checks()
797 if (__kmp_get_ticket_lock_owner(lck) != -1) { in __kmp_destroy_ticket_lock_with_checks()
800 __kmp_destroy_ticket_lock(lck); in __kmp_destroy_ticket_lock_with_checks()
805 int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_nested_ticket_lock() argument
808 if (__kmp_get_ticket_lock_owner(lck) == gtid) { in __kmp_acquire_nested_ticket_lock()
809 std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1, in __kmp_acquire_nested_ticket_lock()
813 __kmp_acquire_ticket_lock_timed_template(lck, gtid); in __kmp_acquire_nested_ticket_lock()
814 std::atomic_store_explicit(&lck->lk.depth_locked, 1, in __kmp_acquire_nested_ticket_lock()
816 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1, in __kmp_acquire_nested_ticket_lock()
822 static int __kmp_acquire_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck, in __kmp_acquire_nested_ticket_lock_with_checks() argument
826 if (!std::atomic_load_explicit(&lck->lk.initialized, in __kmp_acquire_nested_ticket_lock_with_checks()
830 if (lck->lk.self != lck) { in __kmp_acquire_nested_ticket_lock_with_checks()
833 if (!__kmp_is_ticket_lock_nestable(lck)) { in __kmp_acquire_nested_ticket_lock_with_checks()
836 return __kmp_acquire_nested_ticket_lock(lck, gtid); in __kmp_acquire_nested_ticket_lock_with_checks()
839 int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_test_nested_ticket_lock() argument
844 if (__kmp_get_ticket_lock_owner(lck) == gtid) { in __kmp_test_nested_ticket_lock()
845 retval = std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1, in __kmp_test_nested_ticket_lock()
848 } else if (!__kmp_test_ticket_lock(lck, gtid)) { in __kmp_test_nested_ticket_lock()
851 std::atomic_store_explicit(&lck->lk.depth_locked, 1, in __kmp_test_nested_ticket_lock()
853 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1, in __kmp_test_nested_ticket_lock()
860 static int __kmp_test_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck, in __kmp_test_nested_ticket_lock_with_checks() argument
864 if (!std::atomic_load_explicit(&lck->lk.initialized, in __kmp_test_nested_ticket_lock_with_checks()
868 if (lck->lk.self != lck) { in __kmp_test_nested_ticket_lock_with_checks()
871 if (!__kmp_is_ticket_lock_nestable(lck)) { in __kmp_test_nested_ticket_lock_with_checks()
874 return __kmp_test_nested_ticket_lock(lck, gtid); in __kmp_test_nested_ticket_lock_with_checks()
877 int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_release_nested_ticket_lock() argument
880 if ((std::atomic_fetch_add_explicit(&lck->lk.depth_locked, -1, in __kmp_release_nested_ticket_lock()
883 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed); in __kmp_release_nested_ticket_lock()
884 __kmp_release_ticket_lock(lck, gtid); in __kmp_release_nested_ticket_lock()
890 static int __kmp_release_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck, in __kmp_release_nested_ticket_lock_with_checks() argument
894 if (!std::atomic_load_explicit(&lck->lk.initialized, in __kmp_release_nested_ticket_lock_with_checks()
898 if (lck->lk.self != lck) { in __kmp_release_nested_ticket_lock_with_checks()
901 if (!__kmp_is_ticket_lock_nestable(lck)) { in __kmp_release_nested_ticket_lock_with_checks()
904 if (__kmp_get_ticket_lock_owner(lck) == -1) { in __kmp_release_nested_ticket_lock_with_checks()
907 if (__kmp_get_ticket_lock_owner(lck) != gtid) { in __kmp_release_nested_ticket_lock_with_checks()
910 return __kmp_release_nested_ticket_lock(lck, gtid); in __kmp_release_nested_ticket_lock_with_checks()
913 void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck) { in __kmp_init_nested_ticket_lock() argument
914 __kmp_init_ticket_lock(lck); in __kmp_init_nested_ticket_lock()
915 std::atomic_store_explicit(&lck->lk.depth_locked, 0, in __kmp_init_nested_ticket_lock()
920 void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck) { in __kmp_destroy_nested_ticket_lock() argument
921 __kmp_destroy_ticket_lock(lck); in __kmp_destroy_nested_ticket_lock()
922 std::atomic_store_explicit(&lck->lk.depth_locked, 0, in __kmp_destroy_nested_ticket_lock()
927 __kmp_destroy_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) { in __kmp_destroy_nested_ticket_lock_with_checks() argument
930 if (!std::atomic_load_explicit(&lck->lk.initialized, in __kmp_destroy_nested_ticket_lock_with_checks()
934 if (lck->lk.self != lck) { in __kmp_destroy_nested_ticket_lock_with_checks()
937 if (!__kmp_is_ticket_lock_nestable(lck)) { in __kmp_destroy_nested_ticket_lock_with_checks()
940 if (__kmp_get_ticket_lock_owner(lck) != -1) { in __kmp_destroy_nested_ticket_lock_with_checks()
943 __kmp_destroy_nested_ticket_lock(lck); in __kmp_destroy_nested_ticket_lock_with_checks()
948 static const ident_t *__kmp_get_ticket_lock_location(kmp_ticket_lock_t *lck) { in __kmp_get_ticket_lock_location() argument
949 return lck->lk.location; in __kmp_get_ticket_lock_location()
952 static void __kmp_set_ticket_lock_location(kmp_ticket_lock_t *lck, in __kmp_set_ticket_lock_location() argument
954 lck->lk.location = loc; in __kmp_set_ticket_lock_location()
957 static kmp_lock_flags_t __kmp_get_ticket_lock_flags(kmp_ticket_lock_t *lck) { in __kmp_get_ticket_lock_flags() argument
958 return lck->lk.flags; in __kmp_get_ticket_lock_flags()
961 static void __kmp_set_ticket_lock_flags(kmp_ticket_lock_t *lck, in __kmp_set_ticket_lock_flags() argument
963 lck->lk.flags = flags; in __kmp_set_ticket_lock_flags()
1036 kmp_queuing_lock_t *lck, kmp_int32 head_id, in __kmp_dump_queuing_lock() argument
1056 __kmp_printf_no_lock("\t\thead: %d ", lck->lk.head_id); in __kmp_dump_queuing_lock()
1058 if (lck->lk.head_id >= 1) { in __kmp_dump_queuing_lock()
1059 t = __kmp_threads[lck->lk.head_id - 1]->th.th_next_waiting; in __kmp_dump_queuing_lock()
1065 __kmp_printf_no_lock("; tail: %d ", lck->lk.tail_id); in __kmp_dump_queuing_lock()
1071 static kmp_int32 __kmp_get_queuing_lock_owner(kmp_queuing_lock_t *lck) { in __kmp_get_queuing_lock_owner() argument
1072 return TCR_4(lck->lk.owner_id) - 1; in __kmp_get_queuing_lock_owner()
1075 static inline bool __kmp_is_queuing_lock_nestable(kmp_queuing_lock_t *lck) { in __kmp_is_queuing_lock_nestable() argument
1076 return lck->lk.depth_locked != -1; in __kmp_is_queuing_lock_nestable()
1084 __kmp_acquire_queuing_lock_timed_template(kmp_queuing_lock_t *lck, in __kmp_acquire_queuing_lock_timed_template() argument
1087 volatile kmp_int32 *head_id_p = &lck->lk.head_id; in __kmp_acquire_queuing_lock_timed_template()
1088 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id; in __kmp_acquire_queuing_lock_timed_template()
1096 ("__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid)); in __kmp_acquire_queuing_lock_timed_template()
1098 KMP_FSYNC_PREPARE(lck); in __kmp_acquire_queuing_lock_timed_template()
1105 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p); in __kmp_acquire_queuing_lock_timed_template()
1107 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p); in __kmp_acquire_queuing_lock_timed_template()
1192 lck, gtid)); in __kmp_acquire_queuing_lock_timed_template()
1205 KMP_FSYNC_ACQUIRED(lck); in __kmp_acquire_queuing_lock_timed_template()
1216 this_thr->th.ompt_thread_info.wait_id = (uint64_t)lck; in __kmp_acquire_queuing_lock_timed_template()
1230 lck, gtid)); in __kmp_acquire_queuing_lock_timed_template()
1234 KMP_WAIT(spin_here_p, FALSE, KMP_EQ, lck); in __kmp_acquire_queuing_lock_timed_template()
1243 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p); in __kmp_acquire_queuing_lock_timed_template()
1248 lck, gtid)); in __kmp_acquire_queuing_lock_timed_template()
1277 int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_queuing_lock() argument
1280 int retval = __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid); in __kmp_acquire_queuing_lock()
1284 static int __kmp_acquire_queuing_lock_with_checks(kmp_queuing_lock_t *lck, in __kmp_acquire_queuing_lock_with_checks() argument
1287 if (lck->lk.initialized != lck) { in __kmp_acquire_queuing_lock_with_checks()
1290 if (__kmp_is_queuing_lock_nestable(lck)) { in __kmp_acquire_queuing_lock_with_checks()
1293 if (__kmp_get_queuing_lock_owner(lck) == gtid) { in __kmp_acquire_queuing_lock_with_checks()
1297 __kmp_acquire_queuing_lock(lck, gtid); in __kmp_acquire_queuing_lock_with_checks()
1299 lck->lk.owner_id = gtid + 1; in __kmp_acquire_queuing_lock_with_checks()
1303 int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_test_queuing_lock() argument
1304 volatile kmp_int32 *head_id_p = &lck->lk.head_id; in __kmp_test_queuing_lock()
1325 KMP_FSYNC_ACQUIRED(lck); in __kmp_test_queuing_lock()
1335 static int __kmp_test_queuing_lock_with_checks(kmp_queuing_lock_t *lck, in __kmp_test_queuing_lock_with_checks() argument
1338 if (lck->lk.initialized != lck) { in __kmp_test_queuing_lock_with_checks()
1341 if (__kmp_is_queuing_lock_nestable(lck)) { in __kmp_test_queuing_lock_with_checks()
1345 int retval = __kmp_test_queuing_lock(lck, gtid); in __kmp_test_queuing_lock_with_checks()
1348 lck->lk.owner_id = gtid + 1; in __kmp_test_queuing_lock_with_checks()
1353 int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_release_queuing_lock() argument
1354 volatile kmp_int32 *head_id_p = &lck->lk.head_id; in __kmp_release_queuing_lock()
1355 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id; in __kmp_release_queuing_lock()
1358 ("__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid)); in __kmp_release_queuing_lock()
1368 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p); in __kmp_release_queuing_lock()
1370 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p); in __kmp_release_queuing_lock()
1375 KMP_FSYNC_RELEASING(lck); in __kmp_release_queuing_lock()
1388 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail); in __kmp_release_queuing_lock()
1399 lck, gtid)); in __kmp_release_queuing_lock()
1416 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail); in __kmp_release_queuing_lock()
1437 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail); in __kmp_release_queuing_lock()
1461 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail); in __kmp_release_queuing_lock()
1478 lck, gtid)); in __kmp_release_queuing_lock()
1496 static int __kmp_release_queuing_lock_with_checks(kmp_queuing_lock_t *lck, in __kmp_release_queuing_lock_with_checks() argument
1500 if (lck->lk.initialized != lck) { in __kmp_release_queuing_lock_with_checks()
1503 if (__kmp_is_queuing_lock_nestable(lck)) { in __kmp_release_queuing_lock_with_checks()
1506 if (__kmp_get_queuing_lock_owner(lck) == -1) { in __kmp_release_queuing_lock_with_checks()
1509 if (__kmp_get_queuing_lock_owner(lck) != gtid) { in __kmp_release_queuing_lock_with_checks()
1512 lck->lk.owner_id = 0; in __kmp_release_queuing_lock_with_checks()
1513 return __kmp_release_queuing_lock(lck, gtid); in __kmp_release_queuing_lock_with_checks()
1516 void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck) { in __kmp_init_queuing_lock() argument
1517 lck->lk.location = NULL; in __kmp_init_queuing_lock()
1518 lck->lk.head_id = 0; in __kmp_init_queuing_lock()
1519 lck->lk.tail_id = 0; in __kmp_init_queuing_lock()
1520 lck->lk.next_ticket = 0; in __kmp_init_queuing_lock()
1521 lck->lk.now_serving = 0; in __kmp_init_queuing_lock()
1522 lck->lk.owner_id = 0; // no thread owns the lock. in __kmp_init_queuing_lock()
1523 lck->lk.depth_locked = -1; // >= 0 for nestable locks, -1 for simple locks. in __kmp_init_queuing_lock()
1524 lck->lk.initialized = lck; in __kmp_init_queuing_lock()
1526 KA_TRACE(1000, ("__kmp_init_queuing_lock: lock %p initialized\n", lck)); in __kmp_init_queuing_lock()
1529 void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck) { in __kmp_destroy_queuing_lock() argument
1530 lck->lk.initialized = NULL; in __kmp_destroy_queuing_lock()
1531 lck->lk.location = NULL; in __kmp_destroy_queuing_lock()
1532 lck->lk.head_id = 0; in __kmp_destroy_queuing_lock()
1533 lck->lk.tail_id = 0; in __kmp_destroy_queuing_lock()
1534 lck->lk.next_ticket = 0; in __kmp_destroy_queuing_lock()
1535 lck->lk.now_serving = 0; in __kmp_destroy_queuing_lock()
1536 lck->lk.owner_id = 0; in __kmp_destroy_queuing_lock()
1537 lck->lk.depth_locked = -1; in __kmp_destroy_queuing_lock()
1540 static void __kmp_destroy_queuing_lock_with_checks(kmp_queuing_lock_t *lck) { in __kmp_destroy_queuing_lock_with_checks() argument
1542 if (lck->lk.initialized != lck) { in __kmp_destroy_queuing_lock_with_checks()
1545 if (__kmp_is_queuing_lock_nestable(lck)) { in __kmp_destroy_queuing_lock_with_checks()
1548 if (__kmp_get_queuing_lock_owner(lck) != -1) { in __kmp_destroy_queuing_lock_with_checks()
1551 __kmp_destroy_queuing_lock(lck); in __kmp_destroy_queuing_lock_with_checks()
1556 int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_nested_queuing_lock() argument
1559 if (__kmp_get_queuing_lock_owner(lck) == gtid) { in __kmp_acquire_nested_queuing_lock()
1560 lck->lk.depth_locked += 1; in __kmp_acquire_nested_queuing_lock()
1563 __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid); in __kmp_acquire_nested_queuing_lock()
1565 lck->lk.depth_locked = 1; in __kmp_acquire_nested_queuing_lock()
1567 lck->lk.owner_id = gtid + 1; in __kmp_acquire_nested_queuing_lock()
1573 __kmp_acquire_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck, in __kmp_acquire_nested_queuing_lock_with_checks() argument
1576 if (lck->lk.initialized != lck) { in __kmp_acquire_nested_queuing_lock_with_checks()
1579 if (!__kmp_is_queuing_lock_nestable(lck)) { in __kmp_acquire_nested_queuing_lock_with_checks()
1582 return __kmp_acquire_nested_queuing_lock(lck, gtid); in __kmp_acquire_nested_queuing_lock_with_checks()
1585 int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_test_nested_queuing_lock() argument
1590 if (__kmp_get_queuing_lock_owner(lck) == gtid) { in __kmp_test_nested_queuing_lock()
1591 retval = ++lck->lk.depth_locked; in __kmp_test_nested_queuing_lock()
1592 } else if (!__kmp_test_queuing_lock(lck, gtid)) { in __kmp_test_nested_queuing_lock()
1596 retval = lck->lk.depth_locked = 1; in __kmp_test_nested_queuing_lock()
1598 lck->lk.owner_id = gtid + 1; in __kmp_test_nested_queuing_lock()
1603 static int __kmp_test_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck, in __kmp_test_nested_queuing_lock_with_checks() argument
1606 if (lck->lk.initialized != lck) { in __kmp_test_nested_queuing_lock_with_checks()
1609 if (!__kmp_is_queuing_lock_nestable(lck)) { in __kmp_test_nested_queuing_lock_with_checks()
1612 return __kmp_test_nested_queuing_lock(lck, gtid); in __kmp_test_nested_queuing_lock_with_checks()
1615 int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_release_nested_queuing_lock() argument
1619 if (--(lck->lk.depth_locked) == 0) { in __kmp_release_nested_queuing_lock()
1621 lck->lk.owner_id = 0; in __kmp_release_nested_queuing_lock()
1622 __kmp_release_queuing_lock(lck, gtid); in __kmp_release_nested_queuing_lock()
1629 __kmp_release_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck, in __kmp_release_nested_queuing_lock_with_checks() argument
1633 if (lck->lk.initialized != lck) { in __kmp_release_nested_queuing_lock_with_checks()
1636 if (!__kmp_is_queuing_lock_nestable(lck)) { in __kmp_release_nested_queuing_lock_with_checks()
1639 if (__kmp_get_queuing_lock_owner(lck) == -1) { in __kmp_release_nested_queuing_lock_with_checks()
1642 if (__kmp_get_queuing_lock_owner(lck) != gtid) { in __kmp_release_nested_queuing_lock_with_checks()
1645 return __kmp_release_nested_queuing_lock(lck, gtid); in __kmp_release_nested_queuing_lock_with_checks()
1648 void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck) { in __kmp_init_nested_queuing_lock() argument
1649 __kmp_init_queuing_lock(lck); in __kmp_init_nested_queuing_lock()
1650 lck->lk.depth_locked = 0; // >= 0 for nestable locks, -1 for simple locks in __kmp_init_nested_queuing_lock()
1653 void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck) { in __kmp_destroy_nested_queuing_lock() argument
1654 __kmp_destroy_queuing_lock(lck); in __kmp_destroy_nested_queuing_lock()
1655 lck->lk.depth_locked = 0; in __kmp_destroy_nested_queuing_lock()
1659 __kmp_destroy_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) { in __kmp_destroy_nested_queuing_lock_with_checks() argument
1661 if (lck->lk.initialized != lck) { in __kmp_destroy_nested_queuing_lock_with_checks()
1664 if (!__kmp_is_queuing_lock_nestable(lck)) { in __kmp_destroy_nested_queuing_lock_with_checks()
1667 if (__kmp_get_queuing_lock_owner(lck) != -1) { in __kmp_destroy_nested_queuing_lock_with_checks()
1670 __kmp_destroy_nested_queuing_lock(lck); in __kmp_destroy_nested_queuing_lock_with_checks()
1675 static const ident_t *__kmp_get_queuing_lock_location(kmp_queuing_lock_t *lck) { in __kmp_get_queuing_lock_location() argument
1676 return lck->lk.location; in __kmp_get_queuing_lock_location()
1679 static void __kmp_set_queuing_lock_location(kmp_queuing_lock_t *lck, in __kmp_set_queuing_lock_location() argument
1681 lck->lk.location = loc; in __kmp_set_queuing_lock_location()
1684 static kmp_lock_flags_t __kmp_get_queuing_lock_flags(kmp_queuing_lock_t *lck) { in __kmp_get_queuing_lock_flags() argument
1685 return lck->lk.flags; in __kmp_get_queuing_lock_flags()
1688 static void __kmp_set_queuing_lock_flags(kmp_queuing_lock_t *lck, in __kmp_set_queuing_lock_flags() argument
1690 lck->lk.flags = flags; in __kmp_set_queuing_lock_flags()
1813 kmp_adaptive_lock_info_t *lck = &liveLocks; in __kmp_init_speculative_stats() local
1815 memset(CCAST(kmp_adaptive_lock_statistics_t *, &(lck->stats)), 0, in __kmp_init_speculative_stats()
1816 sizeof(lck->stats)); in __kmp_init_speculative_stats()
1817 lck->stats.next = lck; in __kmp_init_speculative_stats()
1818 lck->stats.prev = lck; in __kmp_init_speculative_stats()
1820 KMP_ASSERT(lck->stats.next->stats.prev == lck); in __kmp_init_speculative_stats()
1821 KMP_ASSERT(lck->stats.prev->stats.next == lck); in __kmp_init_speculative_stats()
1827 static void __kmp_remember_lock(kmp_adaptive_lock_info_t *lck) { in __kmp_remember_lock() argument
1830 lck->stats.next = liveLocks.stats.next; in __kmp_remember_lock()
1831 lck->stats.prev = &liveLocks; in __kmp_remember_lock()
1833 liveLocks.stats.next = lck; in __kmp_remember_lock()
1834 lck->stats.next->stats.prev = lck; in __kmp_remember_lock()
1836 KMP_ASSERT(lck->stats.next->stats.prev == lck); in __kmp_remember_lock()
1837 KMP_ASSERT(lck->stats.prev->stats.next == lck); in __kmp_remember_lock()
1842 static void __kmp_forget_lock(kmp_adaptive_lock_info_t *lck) { in __kmp_forget_lock() argument
1843 KMP_ASSERT(lck->stats.next->stats.prev == lck); in __kmp_forget_lock()
1844 KMP_ASSERT(lck->stats.prev->stats.next == lck); in __kmp_forget_lock()
1846 kmp_adaptive_lock_info_t *n = lck->stats.next; in __kmp_forget_lock()
1847 kmp_adaptive_lock_info_t *p = lck->stats.prev; in __kmp_forget_lock()
1853 static void __kmp_zero_speculative_stats(kmp_adaptive_lock_info_t *lck) { in __kmp_zero_speculative_stats() argument
1854 memset(CCAST(kmp_adaptive_lock_statistics_t *, &lck->stats), 0, in __kmp_zero_speculative_stats()
1855 sizeof(lck->stats)); in __kmp_zero_speculative_stats()
1856 __kmp_remember_lock(lck); in __kmp_zero_speculative_stats()
1860 kmp_adaptive_lock_info_t *lck) { in __kmp_add_stats() argument
1861 kmp_adaptive_lock_statistics_t volatile *s = &lck->stats; in __kmp_add_stats()
1863 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts; in __kmp_add_stats()
1871 static void __kmp_accumulate_speculative_stats(kmp_adaptive_lock_info_t *lck) { in __kmp_accumulate_speculative_stats() argument
1874 __kmp_add_stats(&destroyedStats, lck); in __kmp_accumulate_speculative_stats()
1875 __kmp_forget_lock(lck); in __kmp_accumulate_speculative_stats()
1886 kmp_adaptive_lock_info_t *lck; in __kmp_print_speculative_stats() local
1888 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) { in __kmp_print_speculative_stats()
1889 __kmp_add_stats(&total, lck); in __kmp_print_speculative_stats()
1944 #define KMP_INC_STAT(lck, stat) (lck->lk.adaptive.stats.stat++) argument
1946 #define KMP_INC_STAT(lck, stat) argument
1950 static inline bool __kmp_is_unlocked_queuing_lock(kmp_queuing_lock_t *lck) { in __kmp_is_unlocked_queuing_lock() argument
1953 bool res = lck->lk.head_id == 0; in __kmp_is_unlocked_queuing_lock()
1968 __kmp_update_badness_after_success(kmp_adaptive_lock_t *lck) { in __kmp_update_badness_after_success() argument
1970 lck->lk.adaptive.badness = 0; in __kmp_update_badness_after_success()
1971 KMP_INC_STAT(lck, successfulSpeculations); in __kmp_update_badness_after_success()
1975 static __inline void __kmp_step_badness(kmp_adaptive_lock_t *lck) { in __kmp_step_badness() argument
1976 kmp_uint32 newBadness = (lck->lk.adaptive.badness << 1) | 1; in __kmp_step_badness()
1977 if (newBadness > lck->lk.adaptive.max_badness) { in __kmp_step_badness()
1980 lck->lk.adaptive.badness = newBadness; in __kmp_step_badness()
1986 static __inline int __kmp_should_speculate(kmp_adaptive_lock_t *lck, in __kmp_should_speculate() argument
1988 kmp_uint32 badness = lck->lk.adaptive.badness; in __kmp_should_speculate()
1989 kmp_uint32 attempts = lck->lk.adaptive.acquire_attempts; in __kmp_should_speculate()
1997 static int __kmp_test_adaptive_lock_only(kmp_adaptive_lock_t *lck, in __kmp_test_adaptive_lock_only() argument
1999 int retries = lck->lk.adaptive.max_soft_retries; in __kmp_test_adaptive_lock_only()
2016 if (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) { in __kmp_test_adaptive_lock_only()
2026 KMP_INC_STAT(lck, softFailedSpeculations); in __kmp_test_adaptive_lock_only()
2029 KMP_INC_STAT(lck, hardFailedSpeculations); in __kmp_test_adaptive_lock_only()
2038 __kmp_step_badness(lck); in __kmp_test_adaptive_lock_only()
2045 static int __kmp_test_adaptive_lock(kmp_adaptive_lock_t *lck, kmp_int32 gtid) { in __kmp_test_adaptive_lock() argument
2047 if (__kmp_should_speculate(lck, gtid) && in __kmp_test_adaptive_lock()
2048 __kmp_test_adaptive_lock_only(lck, gtid)) in __kmp_test_adaptive_lock()
2053 lck->lk.adaptive.acquire_attempts++; in __kmp_test_adaptive_lock()
2056 if (__kmp_test_queuing_lock(GET_QLK_PTR(lck), gtid)) { in __kmp_test_adaptive_lock()
2057 KMP_INC_STAT(lck, nonSpeculativeAcquires); in __kmp_test_adaptive_lock()
2064 static int __kmp_test_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck, in __kmp_test_adaptive_lock_with_checks() argument
2067 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) { in __kmp_test_adaptive_lock_with_checks()
2071 int retval = __kmp_test_adaptive_lock(lck, gtid); in __kmp_test_adaptive_lock_with_checks()
2074 lck->lk.qlk.owner_id = gtid + 1; in __kmp_test_adaptive_lock_with_checks()
2090 static void __kmp_acquire_adaptive_lock(kmp_adaptive_lock_t *lck, in __kmp_acquire_adaptive_lock() argument
2092 if (__kmp_should_speculate(lck, gtid)) { in __kmp_acquire_adaptive_lock()
2093 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) { in __kmp_acquire_adaptive_lock()
2094 if (__kmp_test_adaptive_lock_only(lck, gtid)) in __kmp_acquire_adaptive_lock()
2103 while (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) { in __kmp_acquire_adaptive_lock()
2104 KMP_INC_STAT(lck, lemmingYields); in __kmp_acquire_adaptive_lock()
2108 if (__kmp_test_adaptive_lock_only(lck, gtid)) in __kmp_acquire_adaptive_lock()
2115 lck->lk.adaptive.acquire_attempts++; in __kmp_acquire_adaptive_lock()
2117 __kmp_acquire_queuing_lock_timed_template<FALSE>(GET_QLK_PTR(lck), gtid); in __kmp_acquire_adaptive_lock()
2119 KMP_INC_STAT(lck, nonSpeculativeAcquires); in __kmp_acquire_adaptive_lock()
2122 static void __kmp_acquire_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck, in __kmp_acquire_adaptive_lock_with_checks() argument
2125 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) { in __kmp_acquire_adaptive_lock_with_checks()
2128 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == gtid) { in __kmp_acquire_adaptive_lock_with_checks()
2132 __kmp_acquire_adaptive_lock(lck, gtid); in __kmp_acquire_adaptive_lock_with_checks()
2134 lck->lk.qlk.owner_id = gtid + 1; in __kmp_acquire_adaptive_lock_with_checks()
2138 static int __kmp_release_adaptive_lock(kmp_adaptive_lock_t *lck, in __kmp_release_adaptive_lock() argument
2141 lck))) { // If the lock doesn't look claimed we must be speculating. in __kmp_release_adaptive_lock()
2145 __kmp_update_badness_after_success(lck); in __kmp_release_adaptive_lock()
2148 __kmp_release_queuing_lock(GET_QLK_PTR(lck), gtid); in __kmp_release_adaptive_lock()
2153 static int __kmp_release_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck, in __kmp_release_adaptive_lock_with_checks() argument
2157 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) { in __kmp_release_adaptive_lock_with_checks()
2160 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == -1) { in __kmp_release_adaptive_lock_with_checks()
2163 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != gtid) { in __kmp_release_adaptive_lock_with_checks()
2166 lck->lk.qlk.owner_id = 0; in __kmp_release_adaptive_lock_with_checks()
2167 __kmp_release_adaptive_lock(lck, gtid); in __kmp_release_adaptive_lock_with_checks()
2171 static void __kmp_init_adaptive_lock(kmp_adaptive_lock_t *lck) { in __kmp_init_adaptive_lock() argument
2172 __kmp_init_queuing_lock(GET_QLK_PTR(lck)); in __kmp_init_adaptive_lock()
2173 lck->lk.adaptive.badness = 0; in __kmp_init_adaptive_lock()
2174 lck->lk.adaptive.acquire_attempts = 0; // nonSpeculativeAcquireAttempts = 0; in __kmp_init_adaptive_lock()
2175 lck->lk.adaptive.max_soft_retries = in __kmp_init_adaptive_lock()
2177 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness; in __kmp_init_adaptive_lock()
2179 __kmp_zero_speculative_stats(&lck->lk.adaptive); in __kmp_init_adaptive_lock()
2181 KA_TRACE(1000, ("__kmp_init_adaptive_lock: lock %p initialized\n", lck)); in __kmp_init_adaptive_lock()
2184 static void __kmp_destroy_adaptive_lock(kmp_adaptive_lock_t *lck) { in __kmp_destroy_adaptive_lock() argument
2186 __kmp_accumulate_speculative_stats(&lck->lk.adaptive); in __kmp_destroy_adaptive_lock()
2188 __kmp_destroy_queuing_lock(GET_QLK_PTR(lck)); in __kmp_destroy_adaptive_lock()
2192 static void __kmp_destroy_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) { in __kmp_destroy_adaptive_lock_with_checks() argument
2194 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) { in __kmp_destroy_adaptive_lock_with_checks()
2197 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != -1) { in __kmp_destroy_adaptive_lock_with_checks()
2200 __kmp_destroy_adaptive_lock(lck); in __kmp_destroy_adaptive_lock_with_checks()
2209 static kmp_int32 __kmp_get_drdpa_lock_owner(kmp_drdpa_lock_t *lck) { in __kmp_get_drdpa_lock_owner() argument
2210 return lck->lk.owner_id - 1; in __kmp_get_drdpa_lock_owner()
2213 static inline bool __kmp_is_drdpa_lock_nestable(kmp_drdpa_lock_t *lck) { in __kmp_is_drdpa_lock_nestable() argument
2214 return lck->lk.depth_locked != -1; in __kmp_is_drdpa_lock_nestable()
2218 __kmp_acquire_drdpa_lock_timed_template(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_drdpa_lock_timed_template() argument
2219 kmp_uint64 ticket = KMP_ATOMIC_INC(&lck->lk.next_ticket); in __kmp_acquire_drdpa_lock_timed_template()
2220 kmp_uint64 mask = lck->lk.mask; // atomic load in __kmp_acquire_drdpa_lock_timed_template()
2221 std::atomic<kmp_uint64> *polls = lck->lk.polls; in __kmp_acquire_drdpa_lock_timed_template()
2225 __kmp_printf("LOCK CONTENTION: %p\n", lck); in __kmp_acquire_drdpa_lock_timed_template()
2238 KMP_FSYNC_PREPARE(lck); in __kmp_acquire_drdpa_lock_timed_template()
2250 mask = lck->lk.mask; // atomic load in __kmp_acquire_drdpa_lock_timed_template()
2251 polls = lck->lk.polls; // atomic load in __kmp_acquire_drdpa_lock_timed_template()
2255 KMP_FSYNC_ACQUIRED(lck); in __kmp_acquire_drdpa_lock_timed_template()
2257 ticket, lck)); in __kmp_acquire_drdpa_lock_timed_template()
2258 lck->lk.now_serving = ticket; // non-volatile store in __kmp_acquire_drdpa_lock_timed_template()
2265 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) { in __kmp_acquire_drdpa_lock_timed_template()
2266 __kmp_free(lck->lk.old_polls); in __kmp_acquire_drdpa_lock_timed_template()
2267 lck->lk.old_polls = NULL; in __kmp_acquire_drdpa_lock_timed_template()
2268 lck->lk.cleanup_ticket = 0; in __kmp_acquire_drdpa_lock_timed_template()
2274 if (lck->lk.old_polls == NULL) { in __kmp_acquire_drdpa_lock_timed_template()
2277 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls); in __kmp_acquire_drdpa_lock_timed_template()
2285 num_polls = TCR_4(lck->lk.num_polls); in __kmp_acquire_drdpa_lock_timed_template()
2296 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1; in __kmp_acquire_drdpa_lock_timed_template()
2330 ticket, lck, num_polls)); in __kmp_acquire_drdpa_lock_timed_template()
2332 lck->lk.old_polls = old_polls; in __kmp_acquire_drdpa_lock_timed_template()
2333 lck->lk.polls = polls; // atomic store in __kmp_acquire_drdpa_lock_timed_template()
2337 lck->lk.num_polls = num_polls; in __kmp_acquire_drdpa_lock_timed_template()
2338 lck->lk.mask = mask; // atomic store in __kmp_acquire_drdpa_lock_timed_template()
2346 lck->lk.cleanup_ticket = lck->lk.next_ticket; in __kmp_acquire_drdpa_lock_timed_template()
2352 int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_drdpa_lock() argument
2353 int retval = __kmp_acquire_drdpa_lock_timed_template(lck, gtid); in __kmp_acquire_drdpa_lock()
2357 static int __kmp_acquire_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck, in __kmp_acquire_drdpa_lock_with_checks() argument
2360 if (lck->lk.initialized != lck) { in __kmp_acquire_drdpa_lock_with_checks()
2363 if (__kmp_is_drdpa_lock_nestable(lck)) { in __kmp_acquire_drdpa_lock_with_checks()
2366 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) == gtid)) { in __kmp_acquire_drdpa_lock_with_checks()
2370 __kmp_acquire_drdpa_lock(lck, gtid); in __kmp_acquire_drdpa_lock_with_checks()
2372 lck->lk.owner_id = gtid + 1; in __kmp_acquire_drdpa_lock_with_checks()
2376 int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_test_drdpa_lock() argument
2379 kmp_uint64 ticket = lck->lk.next_ticket; // atomic load in __kmp_test_drdpa_lock()
2380 std::atomic<kmp_uint64> *polls = lck->lk.polls; in __kmp_test_drdpa_lock()
2381 kmp_uint64 mask = lck->lk.mask; // atomic load in __kmp_test_drdpa_lock()
2384 if (__kmp_atomic_compare_store_acq(&lck->lk.next_ticket, ticket, in __kmp_test_drdpa_lock()
2386 KMP_FSYNC_ACQUIRED(lck); in __kmp_test_drdpa_lock()
2388 ticket, lck)); in __kmp_test_drdpa_lock()
2389 lck->lk.now_serving = ticket; // non-volatile store in __kmp_test_drdpa_lock()
2403 static int __kmp_test_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck, in __kmp_test_drdpa_lock_with_checks() argument
2406 if (lck->lk.initialized != lck) { in __kmp_test_drdpa_lock_with_checks()
2409 if (__kmp_is_drdpa_lock_nestable(lck)) { in __kmp_test_drdpa_lock_with_checks()
2413 int retval = __kmp_test_drdpa_lock(lck, gtid); in __kmp_test_drdpa_lock_with_checks()
2416 lck->lk.owner_id = gtid + 1; in __kmp_test_drdpa_lock_with_checks()
2421 int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_release_drdpa_lock() argument
2424 kmp_uint64 ticket = lck->lk.now_serving + 1; // non-atomic load in __kmp_release_drdpa_lock()
2425 std::atomic<kmp_uint64> *polls = lck->lk.polls; // atomic load in __kmp_release_drdpa_lock()
2426 kmp_uint64 mask = lck->lk.mask; // atomic load in __kmp_release_drdpa_lock()
2428 ticket - 1, lck)); in __kmp_release_drdpa_lock()
2429 KMP_FSYNC_RELEASING(lck); in __kmp_release_drdpa_lock()
2434 static int __kmp_release_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck, in __kmp_release_drdpa_lock_with_checks() argument
2438 if (lck->lk.initialized != lck) { in __kmp_release_drdpa_lock_with_checks()
2441 if (__kmp_is_drdpa_lock_nestable(lck)) { in __kmp_release_drdpa_lock_with_checks()
2444 if (__kmp_get_drdpa_lock_owner(lck) == -1) { in __kmp_release_drdpa_lock_with_checks()
2447 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) >= 0) && in __kmp_release_drdpa_lock_with_checks()
2448 (__kmp_get_drdpa_lock_owner(lck) != gtid)) { in __kmp_release_drdpa_lock_with_checks()
2451 lck->lk.owner_id = 0; in __kmp_release_drdpa_lock_with_checks()
2452 return __kmp_release_drdpa_lock(lck, gtid); in __kmp_release_drdpa_lock_with_checks()
2455 void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck) { in __kmp_init_drdpa_lock() argument
2456 lck->lk.location = NULL; in __kmp_init_drdpa_lock()
2457 lck->lk.mask = 0; in __kmp_init_drdpa_lock()
2458 lck->lk.num_polls = 1; in __kmp_init_drdpa_lock()
2459 lck->lk.polls = (std::atomic<kmp_uint64> *)__kmp_allocate( in __kmp_init_drdpa_lock()
2460 lck->lk.num_polls * sizeof(*(lck->lk.polls))); in __kmp_init_drdpa_lock()
2461 lck->lk.cleanup_ticket = 0; in __kmp_init_drdpa_lock()
2462 lck->lk.old_polls = NULL; in __kmp_init_drdpa_lock()
2463 lck->lk.next_ticket = 0; in __kmp_init_drdpa_lock()
2464 lck->lk.now_serving = 0; in __kmp_init_drdpa_lock()
2465 lck->lk.owner_id = 0; // no thread owns the lock. in __kmp_init_drdpa_lock()
2466 lck->lk.depth_locked = -1; // >= 0 for nestable locks, -1 for simple locks. in __kmp_init_drdpa_lock()
2467 lck->lk.initialized = lck; in __kmp_init_drdpa_lock()
2469 KA_TRACE(1000, ("__kmp_init_drdpa_lock: lock %p initialized\n", lck)); in __kmp_init_drdpa_lock()
2472 void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck) { in __kmp_destroy_drdpa_lock() argument
2473 lck->lk.initialized = NULL; in __kmp_destroy_drdpa_lock()
2474 lck->lk.location = NULL; in __kmp_destroy_drdpa_lock()
2475 if (lck->lk.polls.load() != NULL) { in __kmp_destroy_drdpa_lock()
2476 __kmp_free(lck->lk.polls.load()); in __kmp_destroy_drdpa_lock()
2477 lck->lk.polls = NULL; in __kmp_destroy_drdpa_lock()
2479 if (lck->lk.old_polls != NULL) { in __kmp_destroy_drdpa_lock()
2480 __kmp_free(lck->lk.old_polls); in __kmp_destroy_drdpa_lock()
2481 lck->lk.old_polls = NULL; in __kmp_destroy_drdpa_lock()
2483 lck->lk.mask = 0; in __kmp_destroy_drdpa_lock()
2484 lck->lk.num_polls = 0; in __kmp_destroy_drdpa_lock()
2485 lck->lk.cleanup_ticket = 0; in __kmp_destroy_drdpa_lock()
2486 lck->lk.next_ticket = 0; in __kmp_destroy_drdpa_lock()
2487 lck->lk.now_serving = 0; in __kmp_destroy_drdpa_lock()
2488 lck->lk.owner_id = 0; in __kmp_destroy_drdpa_lock()
2489 lck->lk.depth_locked = -1; in __kmp_destroy_drdpa_lock()
2492 static void __kmp_destroy_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) { in __kmp_destroy_drdpa_lock_with_checks() argument
2494 if (lck->lk.initialized != lck) { in __kmp_destroy_drdpa_lock_with_checks()
2497 if (__kmp_is_drdpa_lock_nestable(lck)) { in __kmp_destroy_drdpa_lock_with_checks()
2500 if (__kmp_get_drdpa_lock_owner(lck) != -1) { in __kmp_destroy_drdpa_lock_with_checks()
2503 __kmp_destroy_drdpa_lock(lck); in __kmp_destroy_drdpa_lock_with_checks()
2508 int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_nested_drdpa_lock() argument
2511 if (__kmp_get_drdpa_lock_owner(lck) == gtid) { in __kmp_acquire_nested_drdpa_lock()
2512 lck->lk.depth_locked += 1; in __kmp_acquire_nested_drdpa_lock()
2515 __kmp_acquire_drdpa_lock_timed_template(lck, gtid); in __kmp_acquire_nested_drdpa_lock()
2517 lck->lk.depth_locked = 1; in __kmp_acquire_nested_drdpa_lock()
2519 lck->lk.owner_id = gtid + 1; in __kmp_acquire_nested_drdpa_lock()
2524 static void __kmp_acquire_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck, in __kmp_acquire_nested_drdpa_lock_with_checks() argument
2527 if (lck->lk.initialized != lck) { in __kmp_acquire_nested_drdpa_lock_with_checks()
2530 if (!__kmp_is_drdpa_lock_nestable(lck)) { in __kmp_acquire_nested_drdpa_lock_with_checks()
2533 __kmp_acquire_nested_drdpa_lock(lck, gtid); in __kmp_acquire_nested_drdpa_lock_with_checks()
2536 int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_test_nested_drdpa_lock() argument
2541 if (__kmp_get_drdpa_lock_owner(lck) == gtid) { in __kmp_test_nested_drdpa_lock()
2542 retval = ++lck->lk.depth_locked; in __kmp_test_nested_drdpa_lock()
2543 } else if (!__kmp_test_drdpa_lock(lck, gtid)) { in __kmp_test_nested_drdpa_lock()
2547 retval = lck->lk.depth_locked = 1; in __kmp_test_nested_drdpa_lock()
2549 lck->lk.owner_id = gtid + 1; in __kmp_test_nested_drdpa_lock()
2554 static int __kmp_test_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck, in __kmp_test_nested_drdpa_lock_with_checks() argument
2557 if (lck->lk.initialized != lck) { in __kmp_test_nested_drdpa_lock_with_checks()
2560 if (!__kmp_is_drdpa_lock_nestable(lck)) { in __kmp_test_nested_drdpa_lock_with_checks()
2563 return __kmp_test_nested_drdpa_lock(lck, gtid); in __kmp_test_nested_drdpa_lock_with_checks()
2566 int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_release_nested_drdpa_lock() argument
2570 if (--(lck->lk.depth_locked) == 0) { in __kmp_release_nested_drdpa_lock()
2572 lck->lk.owner_id = 0; in __kmp_release_nested_drdpa_lock()
2573 __kmp_release_drdpa_lock(lck, gtid); in __kmp_release_nested_drdpa_lock()
2579 static int __kmp_release_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck, in __kmp_release_nested_drdpa_lock_with_checks() argument
2583 if (lck->lk.initialized != lck) { in __kmp_release_nested_drdpa_lock_with_checks()
2586 if (!__kmp_is_drdpa_lock_nestable(lck)) { in __kmp_release_nested_drdpa_lock_with_checks()
2589 if (__kmp_get_drdpa_lock_owner(lck) == -1) { in __kmp_release_nested_drdpa_lock_with_checks()
2592 if (__kmp_get_drdpa_lock_owner(lck) != gtid) { in __kmp_release_nested_drdpa_lock_with_checks()
2595 return __kmp_release_nested_drdpa_lock(lck, gtid); in __kmp_release_nested_drdpa_lock_with_checks()
2598 void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck) { in __kmp_init_nested_drdpa_lock() argument
2599 __kmp_init_drdpa_lock(lck); in __kmp_init_nested_drdpa_lock()
2600 lck->lk.depth_locked = 0; // >= 0 for nestable locks, -1 for simple locks in __kmp_init_nested_drdpa_lock()
2603 void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck) { in __kmp_destroy_nested_drdpa_lock() argument
2604 __kmp_destroy_drdpa_lock(lck); in __kmp_destroy_nested_drdpa_lock()
2605 lck->lk.depth_locked = 0; in __kmp_destroy_nested_drdpa_lock()
2608 static void __kmp_destroy_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) { in __kmp_destroy_nested_drdpa_lock_with_checks() argument
2610 if (lck->lk.initialized != lck) { in __kmp_destroy_nested_drdpa_lock_with_checks()
2613 if (!__kmp_is_drdpa_lock_nestable(lck)) { in __kmp_destroy_nested_drdpa_lock_with_checks()
2616 if (__kmp_get_drdpa_lock_owner(lck) != -1) { in __kmp_destroy_nested_drdpa_lock_with_checks()
2619 __kmp_destroy_nested_drdpa_lock(lck); in __kmp_destroy_nested_drdpa_lock_with_checks()
2624 static const ident_t *__kmp_get_drdpa_lock_location(kmp_drdpa_lock_t *lck) { in __kmp_get_drdpa_lock_location() argument
2625 return lck->lk.location; in __kmp_get_drdpa_lock_location()
2628 static void __kmp_set_drdpa_lock_location(kmp_drdpa_lock_t *lck, in __kmp_set_drdpa_lock_location() argument
2630 lck->lk.location = loc; in __kmp_set_drdpa_lock_location()
2633 static kmp_lock_flags_t __kmp_get_drdpa_lock_flags(kmp_drdpa_lock_t *lck) { in __kmp_get_drdpa_lock_flags() argument
2634 return lck->lk.flags; in __kmp_get_drdpa_lock_flags()
2637 static void __kmp_set_drdpa_lock_flags(kmp_drdpa_lock_t *lck, in __kmp_set_drdpa_lock_flags() argument
2639 lck->lk.flags = flags; in __kmp_set_drdpa_lock_flags()
2690 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck, in __kmp_init_direct_lock() argument
2692 TCW_4(*lck, KMP_GET_D_TAG(seq)); in __kmp_init_direct_lock()
2709 static void __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck) { TCW_4(*lck, 0); } in __kmp_destroy_hle_lock() argument
2711 static void __kmp_destroy_hle_lock_with_checks(kmp_dyna_lock_t *lck) { in __kmp_destroy_hle_lock_with_checks() argument
2712 TCW_4(*lck, 0); in __kmp_destroy_hle_lock_with_checks()
2715 static void __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_hle_lock() argument
2717 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) { in __kmp_acquire_hle_lock()
2720 while (*(kmp_uint32 volatile *)lck != KMP_LOCK_FREE(hle)) { in __kmp_acquire_hle_lock()
2725 } while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)); in __kmp_acquire_hle_lock()
2729 static void __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck, in __kmp_acquire_hle_lock_with_checks() argument
2731 __kmp_acquire_hle_lock(lck, gtid); // TODO: add checks in __kmp_acquire_hle_lock_with_checks()
2734 static int __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) { in __kmp_release_hle_lock() argument
2736 : "=m"(*lck) in __kmp_release_hle_lock()
2742 static int __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck, in __kmp_release_hle_lock_with_checks() argument
2744 return __kmp_release_hle_lock(lck, gtid); // TODO: add checks in __kmp_release_hle_lock_with_checks()
2747 static int __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) { in __kmp_test_hle_lock() argument
2748 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle); in __kmp_test_hle_lock()
2751 static int __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck, in __kmp_test_hle_lock_with_checks() argument
2753 return __kmp_test_hle_lock(lck, gtid); // TODO: add checks in __kmp_test_hle_lock_with_checks()
2756 static void __kmp_init_rtm_queuing_lock(kmp_queuing_lock_t *lck) { in __kmp_init_rtm_queuing_lock() argument
2757 __kmp_init_queuing_lock(lck); in __kmp_init_rtm_queuing_lock()
2760 static void __kmp_destroy_rtm_queuing_lock(kmp_queuing_lock_t *lck) { in __kmp_destroy_rtm_queuing_lock() argument
2761 __kmp_destroy_queuing_lock(lck); in __kmp_destroy_rtm_queuing_lock()
2765 __kmp_destroy_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck) { in __kmp_destroy_rtm_queuing_lock_with_checks() argument
2766 __kmp_destroy_queuing_lock_with_checks(lck); in __kmp_destroy_rtm_queuing_lock_with_checks()
2770 static void __kmp_acquire_rtm_queuing_lock(kmp_queuing_lock_t *lck, in __kmp_acquire_rtm_queuing_lock() argument
2776 if (__kmp_is_unlocked_queuing_lock(lck)) in __kmp_acquire_rtm_queuing_lock()
2782 while (!__kmp_is_unlocked_queuing_lock(lck)) { in __kmp_acquire_rtm_queuing_lock()
2790 __kmp_acquire_queuing_lock(lck, gtid); in __kmp_acquire_rtm_queuing_lock()
2793 static void __kmp_acquire_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck, in __kmp_acquire_rtm_queuing_lock_with_checks() argument
2795 __kmp_acquire_rtm_queuing_lock(lck, gtid); in __kmp_acquire_rtm_queuing_lock_with_checks()
2799 static int __kmp_release_rtm_queuing_lock(kmp_queuing_lock_t *lck, in __kmp_release_rtm_queuing_lock() argument
2801 if (__kmp_is_unlocked_queuing_lock(lck)) { in __kmp_release_rtm_queuing_lock()
2806 __kmp_release_queuing_lock(lck, gtid); in __kmp_release_rtm_queuing_lock()
2811 static int __kmp_release_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck, in __kmp_release_rtm_queuing_lock_with_checks() argument
2813 return __kmp_release_rtm_queuing_lock(lck, gtid); in __kmp_release_rtm_queuing_lock_with_checks()
2817 static int __kmp_test_rtm_queuing_lock(kmp_queuing_lock_t *lck, in __kmp_test_rtm_queuing_lock() argument
2822 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) { in __kmp_test_rtm_queuing_lock()
2829 return __kmp_test_queuing_lock(lck, gtid); in __kmp_test_rtm_queuing_lock()
2832 static int __kmp_test_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck, in __kmp_test_rtm_queuing_lock_with_checks() argument
2834 return __kmp_test_rtm_queuing_lock(lck, gtid); in __kmp_test_rtm_queuing_lock_with_checks()
2840 static void __kmp_destroy_rtm_spin_lock(kmp_rtm_spin_lock_t *lck) { in __kmp_destroy_rtm_spin_lock() argument
2841 KMP_ATOMIC_ST_REL(&lck->lk.poll, 0); in __kmp_destroy_rtm_spin_lock()
2844 static void __kmp_destroy_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck) { in __kmp_destroy_rtm_spin_lock_with_checks() argument
2845 __kmp_destroy_rtm_spin_lock(lck); in __kmp_destroy_rtm_spin_lock_with_checks()
2849 static int __kmp_acquire_rtm_spin_lock(kmp_rtm_spin_lock_t *lck, in __kmp_acquire_rtm_spin_lock() argument
2857 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free) in __kmp_acquire_rtm_spin_lock()
2863 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free) { in __kmp_acquire_rtm_spin_lock()
2871 KMP_FSYNC_PREPARE(lck); in __kmp_acquire_rtm_spin_lock()
2873 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free || in __kmp_acquire_rtm_spin_lock()
2874 !__kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) { in __kmp_acquire_rtm_spin_lock()
2877 KMP_FSYNC_ACQUIRED(lck); in __kmp_acquire_rtm_spin_lock()
2881 static int __kmp_acquire_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck, in __kmp_acquire_rtm_spin_lock_with_checks() argument
2883 return __kmp_acquire_rtm_spin_lock(lck, gtid); in __kmp_acquire_rtm_spin_lock_with_checks()
2887 static int __kmp_release_rtm_spin_lock(kmp_rtm_spin_lock_t *lck, in __kmp_release_rtm_spin_lock() argument
2889 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == KMP_LOCK_FREE(rtm_spin)) { in __kmp_release_rtm_spin_lock()
2894 KMP_FSYNC_RELEASING(lck); in __kmp_release_rtm_spin_lock()
2895 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(rtm_spin)); in __kmp_release_rtm_spin_lock()
2900 static int __kmp_release_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck, in __kmp_release_rtm_spin_lock_with_checks() argument
2902 return __kmp_release_rtm_spin_lock(lck, gtid); in __kmp_release_rtm_spin_lock_with_checks()
2906 static int __kmp_test_rtm_spin_lock(kmp_rtm_spin_lock_t *lck, kmp_int32 gtid) { in __kmp_test_rtm_spin_lock() argument
2913 KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free) { in __kmp_test_rtm_spin_lock()
2920 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free && in __kmp_test_rtm_spin_lock()
2921 __kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) { in __kmp_test_rtm_spin_lock()
2922 KMP_FSYNC_ACQUIRED(lck); in __kmp_test_rtm_spin_lock()
2928 static int __kmp_test_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck, in __kmp_test_rtm_spin_lock_with_checks() argument
2930 return __kmp_test_rtm_spin_lock(lck, gtid); in __kmp_test_rtm_spin_lock_with_checks()
3123 kmp_indirect_lock_t *lck; in __kmp_allocate_indirect_lock() local
3130 lck = __kmp_indirect_lock_pool[tag]; in __kmp_allocate_indirect_lock()
3132 idx = lck->lock->pool.index; in __kmp_allocate_indirect_lock()
3133 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next; in __kmp_allocate_indirect_lock()
3135 lck)); in __kmp_allocate_indirect_lock()
3171 lck = &lock_table->table[row][col]; in __kmp_allocate_indirect_lock()
3173 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]); in __kmp_allocate_indirect_lock()
3175 ("__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck)); in __kmp_allocate_indirect_lock()
3180 lck->type = tag; in __kmp_allocate_indirect_lock()
3186 *((kmp_indirect_lock_t **)user_lock) = lck; in __kmp_allocate_indirect_lock()
3189 return lck; in __kmp_allocate_indirect_lock()
3196 kmp_indirect_lock_t *lck = NULL; in __kmp_lookup_indirect_lock() local
3202 lck = __kmp_get_i_lock(idx); in __kmp_lookup_indirect_lock()
3204 lck = *((kmp_indirect_lock_t **)user_lock); in __kmp_lookup_indirect_lock()
3206 if (lck == NULL) { in __kmp_lookup_indirect_lock()
3209 return lck; in __kmp_lookup_indirect_lock()
3299 kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq) { in __kmp_get_user_lock_owner() argument
3303 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck); in __kmp_get_user_lock_owner()
3307 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck); in __kmp_get_user_lock_owner()
3311 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck); in __kmp_get_user_lock_owner()
3317 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck); in __kmp_get_user_lock_owner()
3320 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck); in __kmp_get_user_lock_owner()
3475 static void __kmp_init_tas_lock_with_checks(kmp_tas_lock_t *lck) { in __kmp_init_tas_lock_with_checks() argument
3476 __kmp_init_tas_lock(lck); in __kmp_init_tas_lock_with_checks()
3479 static void __kmp_init_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) { in __kmp_init_nested_tas_lock_with_checks() argument
3480 __kmp_init_nested_tas_lock(lck); in __kmp_init_nested_tas_lock_with_checks()
3484 static void __kmp_init_futex_lock_with_checks(kmp_futex_lock_t *lck) { in __kmp_init_futex_lock_with_checks() argument
3485 __kmp_init_futex_lock(lck); in __kmp_init_futex_lock_with_checks()
3488 static void __kmp_init_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) { in __kmp_init_nested_futex_lock_with_checks() argument
3489 __kmp_init_nested_futex_lock(lck); in __kmp_init_nested_futex_lock_with_checks()
3493 static int __kmp_is_ticket_lock_initialized(kmp_ticket_lock_t *lck) { in __kmp_is_ticket_lock_initialized() argument
3494 return lck == lck->lk.self; in __kmp_is_ticket_lock_initialized()
3497 static void __kmp_init_ticket_lock_with_checks(kmp_ticket_lock_t *lck) { in __kmp_init_ticket_lock_with_checks() argument
3498 __kmp_init_ticket_lock(lck); in __kmp_init_ticket_lock_with_checks()
3501 static void __kmp_init_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) { in __kmp_init_nested_ticket_lock_with_checks() argument
3502 __kmp_init_nested_ticket_lock(lck); in __kmp_init_nested_ticket_lock_with_checks()
3505 static int __kmp_is_queuing_lock_initialized(kmp_queuing_lock_t *lck) { in __kmp_is_queuing_lock_initialized() argument
3506 return lck == lck->lk.initialized; in __kmp_is_queuing_lock_initialized()
3509 static void __kmp_init_queuing_lock_with_checks(kmp_queuing_lock_t *lck) { in __kmp_init_queuing_lock_with_checks() argument
3510 __kmp_init_queuing_lock(lck); in __kmp_init_queuing_lock_with_checks()
3514 __kmp_init_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) { in __kmp_init_nested_queuing_lock_with_checks() argument
3515 __kmp_init_nested_queuing_lock(lck); in __kmp_init_nested_queuing_lock_with_checks()
3519 static void __kmp_init_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) { in __kmp_init_adaptive_lock_with_checks() argument
3520 __kmp_init_adaptive_lock(lck); in __kmp_init_adaptive_lock_with_checks()
3524 static int __kmp_is_drdpa_lock_initialized(kmp_drdpa_lock_t *lck) { in __kmp_is_drdpa_lock_initialized() argument
3525 return lck == lck->lk.initialized; in __kmp_is_drdpa_lock_initialized()
3528 static void __kmp_init_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) { in __kmp_init_drdpa_lock_with_checks() argument
3529 __kmp_init_drdpa_lock(lck); in __kmp_init_drdpa_lock_with_checks()
3532 static void __kmp_init_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) { in __kmp_init_nested_drdpa_lock_with_checks() argument
3533 __kmp_init_nested_drdpa_lock(lck); in __kmp_init_nested_drdpa_lock_with_checks()
3545 kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck) = NULL;
3546 int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
3549 int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
3551 int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
3553 void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3554 void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck) = NULL;
3555 void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3556 int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3559 int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3561 int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3563 void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3564 void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3566 int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck) = NULL;
3567 const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck) = NULL;
3568 void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
3570 kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck) = NULL;
3571 void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
3795 static kmp_lock_index_t __kmp_lock_table_insert(kmp_user_lock_p lck) { in __kmp_lock_table_insert() argument
3821 __kmp_user_lock_table.table[index] = lck; in __kmp_lock_table_insert()
3856 kmp_user_lock_p lck; in __kmp_user_lock_allocate() local
3866 lck = (kmp_user_lock_p)__kmp_allocate(__kmp_user_lock_size); in __kmp_user_lock_allocate()
3868 lck = __kmp_lock_block_allocate(); in __kmp_user_lock_allocate()
3873 index = __kmp_lock_table_insert(lck); in __kmp_user_lock_allocate()
3876 lck = __kmp_lock_pool; in __kmp_user_lock_allocate()
3886 *((kmp_user_lock_p *)user_lock) = lck; in __kmp_user_lock_allocate()
3890 __kmp_set_user_lock_flags(lck, flags); in __kmp_user_lock_allocate()
3894 return lck; in __kmp_user_lock_allocate()
3899 kmp_user_lock_p lck) { in __kmp_user_lock_free() argument
3901 KMP_DEBUG_ASSERT(lck != NULL); in __kmp_user_lock_free()
3905 lck->pool.next = __kmp_lock_pool; in __kmp_user_lock_free()
3906 __kmp_lock_pool = lck; in __kmp_user_lock_free()
3910 lck->pool.index = index; in __kmp_user_lock_free()
3917 kmp_user_lock_p lck = NULL; in __kmp_lookup_user_lock() local
3934 lck = __kmp_user_lock_table.table[index]; in __kmp_lookup_user_lock()
3936 lck = *((kmp_user_lock_p *)user_lock); in __kmp_lookup_user_lock()
3940 if (lck == NULL) { in __kmp_lookup_user_lock()
3945 return lck; in __kmp_lookup_user_lock()
3953 #define IS_CRITICAL(lck) \ in __kmp_cleanup_user_locks() argument
3955 ((*__kmp_get_user_lock_flags_)(lck)&kmp_lf_critical_section)) in __kmp_cleanup_user_locks()
3985 kmp_user_lock_p lck = in __kmp_cleanup_user_locks() local
3989 (*__kmp_is_user_lock_initialized_)(lck)) { in __kmp_cleanup_user_locks()
3993 if (__kmp_env_consistency_check && (!IS_CRITICAL(lck)) && in __kmp_cleanup_user_locks()
3994 ((loc = __kmp_get_user_lock_location(lck)) != NULL) && in __kmp_cleanup_user_locks()
4002 if (IS_CRITICAL(lck)) { in __kmp_cleanup_user_locks()
4006 lck, *(void **)lck)); in __kmp_cleanup_user_locks()
4008 KA_TRACE(20, ("__kmp_cleanup_user_locks: free lock %p (%p)\n", lck, in __kmp_cleanup_user_locks()
4009 *(void **)lck)); in __kmp_cleanup_user_locks()
4014 __kmp_destroy_user_lock(lck); in __kmp_cleanup_user_locks()
4019 __kmp_free(lck); in __kmp_cleanup_user_locks()