Lines Matching refs:th
47 static void bpool(kmp_info_t *th, void *buffer, bufsize len);
48 static void *bget(kmp_info_t *th, bufsize size);
49 static void *bgetz(kmp_info_t *th, bufsize size);
50 static void *bgetr(kmp_info_t *th, void *buffer, bufsize newsize);
51 static void brel(kmp_info_t *th, void *buf);
52 static void bectl(kmp_info_t *th, bget_compact_t compact,
228 static void set_thr_data(kmp_info_t *th) { in set_thr_data() argument
232 data = (thr_data_t *)((!th->th.th_local.bget_data) in set_thr_data()
234 : th->th.th_local.bget_data); in set_thr_data()
243 th->th.th_local.bget_data = data; in set_thr_data()
244 th->th.th_local.bget_list = 0; in set_thr_data()
247 __kmp_init_lock(&th->th.th_local.bget_lock); in set_thr_data()
249 __kmp_init_bootstrap_lock(&th->th.th_local.bget_lock); in set_thr_data()
254 static thr_data_t *get_thr_data(kmp_info_t *th) { in get_thr_data() argument
257 data = (thr_data_t *)th->th.th_local.bget_data; in get_thr_data()
265 static void __kmp_bget_dequeue(kmp_info_t *th) { in __kmp_bget_dequeue() argument
266 void *p = TCR_SYNC_PTR(th->th.th_local.bget_list); in __kmp_bget_dequeue()
271 volatile void *old_value = TCR_SYNC_PTR(th->th.th_local.bget_list); in __kmp_bget_dequeue()
272 while (!KMP_COMPARE_AND_STORE_PTR(&th->th.th_local.bget_list, in __kmp_bget_dequeue()
275 old_value = TCR_SYNC_PTR(th->th.th_local.bget_list); in __kmp_bget_dequeue()
281 __kmp_acquire_lock(&th->th.th_local.bget_lock, __kmp_gtid_from_thread(th)); in __kmp_bget_dequeue()
283 __kmp_acquire_bootstrap_lock(&th->th.th_local.bget_lock); in __kmp_bget_dequeue()
286 p = (void *)th->th.th_local.bget_list; in __kmp_bget_dequeue()
287 th->th.th_local.bget_list = 0; in __kmp_bget_dequeue()
290 __kmp_release_lock(&th->th.th_local.bget_lock, __kmp_gtid_from_thread(th)); in __kmp_bget_dequeue()
292 __kmp_release_bootstrap_lock(&th->th.th_local.bget_lock); in __kmp_bget_dequeue()
303 (kmp_uintptr_t)th); // clear possible mark in __kmp_bget_dequeue()
308 brel(th, buf); in __kmp_bget_dequeue()
314 static void __kmp_bget_enqueue(kmp_info_t *th, void *buf in __kmp_bget_enqueue() argument
324 (kmp_uintptr_t)th); // clear possible mark in __kmp_bget_enqueue()
329 __kmp_gtid_from_thread(th))); in __kmp_bget_enqueue()
333 volatile void *old_value = TCR_PTR(th->th.th_local.bget_list); in __kmp_bget_enqueue()
338 while (!KMP_COMPARE_AND_STORE_PTR(&th->th.th_local.bget_list, in __kmp_bget_enqueue()
341 old_value = TCR_PTR(th->th.th_local.bget_list); in __kmp_bget_enqueue()
349 __kmp_acquire_lock(&th->th.th_local.bget_lock, rel_gtid); in __kmp_bget_enqueue()
351 __kmp_acquire_bootstrap_lock(&th->th.th_local.bget_lock); in __kmp_bget_enqueue()
354 b->ql.flink = BFH(th->th.th_local.bget_list); in __kmp_bget_enqueue()
355 th->th.th_local.bget_list = (void *)buf; in __kmp_bget_enqueue()
358 __kmp_release_lock(&th->th.th_local.bget_lock, rel_gtid); in __kmp_bget_enqueue()
360 __kmp_release_bootstrap_lock(&th->th.th_local.bget_lock); in __kmp_bget_enqueue()
396 static void bcheck(kmp_info_t *th, bufsize *max_free, bufsize *total_free) { in bcheck() argument
397 thr_data_t *thr = get_thr_data(th); in bcheck()
426 static void *bget(kmp_info_t *th, bufsize requested_size) { in bget() argument
427 thr_data_t *thr = get_thr_data(th); in bget()
440 __kmp_bget_dequeue(th); /* Release any queued buffers */ in bget()
516 th); // not an allocated address (do not mark it) in bget()
549 TCW_PTR(ba->bb.bthr, th); // not an allocated address (do not mark) in bget()
596 TCW_PTR(bdh->bh.bb.bthr, th); // don't mark buffer as allocated, in bget()
621 bpool(th, newpool, thr->exp_incr); in bget()
623 th, requested_size); /* This can't, I say, can't get into a loop. */ in bget()
638 static void *bgetz(kmp_info_t *th, bufsize size) { in bgetz() argument
639 char *buf = (char *)bget(th, size); in bgetz()
668 static void *bgetr(kmp_info_t *th, void *buf, bufsize size) { in bgetr() argument
673 nbuf = bget(th, size); in bgetr()
696 brel(th, buf); in bgetr()
702 static void brel(kmp_info_t *th, void *buf) { in brel() argument
703 thr_data_t *thr = get_thr_data(th); in brel()
735 if (bth != th) { in brel()
740 __kmp_gtid_from_thread(th) in brel()
866 static void bectl(kmp_info_t *th, bget_compact_t compact, in bectl() argument
869 thr_data_t *thr = get_thr_data(th); in bectl()
878 static void bpool(kmp_info_t *th, void *buf, bufsize len) { in bpool() argument
880 thr_data_t *thr = get_thr_data(th); in bpool()
884 __kmp_bget_dequeue(th); /* Release any queued buffers */ in bpool()
921 (kmp_info_t *)((kmp_uintptr_t)th | in bpool()
940 static void bfreed(kmp_info_t *th) { in bfreed() argument
942 int gtid = __kmp_gtid_from_thread(th); in bfreed()
943 thr_data_t *thr = get_thr_data(th); in bfreed()
993 void __kmp_initialize_bget(kmp_info_t *th) { in __kmp_initialize_bget() argument
994 KMP_DEBUG_ASSERT(SizeQuant >= sizeof(void *) && (th != 0)); in __kmp_initialize_bget()
996 set_thr_data(th); in __kmp_initialize_bget()
998 bectl(th, (bget_compact_t)0, (bget_acquire_t)malloc, (bget_release_t)free, in __kmp_initialize_bget()
1002 void __kmp_finalize_bget(kmp_info_t *th) { in __kmp_finalize_bget() argument
1006 KMP_DEBUG_ASSERT(th != 0); in __kmp_finalize_bget()
1009 thr = (thr_data_t *)th->th.th_local.bget_data; in __kmp_finalize_bget()
1038 if (th->th.th_local.bget_data != NULL) { in __kmp_finalize_bget()
1039 __kmp_free(th->th.th_local.bget_data); in __kmp_finalize_bget()
1040 th->th.th_local.bget_data = NULL; in __kmp_finalize_bget()
1076 kmp_info_t *th = __kmp_get_thread(); in kmpc_get_poolstat() local
1079 __kmp_bget_dequeue(th); /* Release any queued buffers */ in kmpc_get_poolstat()
1081 bcheck(th, &a, &b); in kmpc_get_poolstat()
1088 kmp_info_t *th = __kmp_get_thread(); in kmpc_poolprint() local
1090 __kmp_bget_dequeue(th); /* Release any queued buffers */ in kmpc_poolprint()
1092 bfreed(th); in kmpc_poolprint()
1178 kmp_info_t *th = __kmp_get_thread(); in kmpc_free() local
1179 __kmp_bget_dequeue(th); /* Release any queued buffers */ in kmpc_free()
1182 brel(th, *((void **)ptr - 1)); in kmpc_free()
1186 void *___kmp_thread_malloc(kmp_info_t *th, size_t size KMP_SRC_LOC_DECL) { in ___kmp_thread_malloc() argument
1188 KE_TRACE(30, ("-> __kmp_thread_malloc( %p, %d ) called from %s:%d\n", th, in ___kmp_thread_malloc()
1190 ptr = bget(th, (bufsize)size); in ___kmp_thread_malloc()
1195 void *___kmp_thread_calloc(kmp_info_t *th, size_t nelem, in ___kmp_thread_calloc() argument
1198 KE_TRACE(30, ("-> __kmp_thread_calloc( %p, %d, %d ) called from %s:%d\n", th, in ___kmp_thread_calloc()
1200 ptr = bgetz(th, (bufsize)(nelem * elsize)); in ___kmp_thread_calloc()
1205 void *___kmp_thread_realloc(kmp_info_t *th, void *ptr, in ___kmp_thread_realloc() argument
1207 KE_TRACE(30, ("-> __kmp_thread_realloc( %p, %p, %d ) called from %s:%d\n", th, in ___kmp_thread_realloc()
1209 ptr = bgetr(th, ptr, (bufsize)size); in ___kmp_thread_realloc()
1214 void ___kmp_thread_free(kmp_info_t *th, void *ptr KMP_SRC_LOC_DECL) { in ___kmp_thread_free() argument
1215 KE_TRACE(30, ("-> __kmp_thread_free( %p, %p ) called from %s:%d\n", th, in ___kmp_thread_free()
1218 __kmp_bget_dequeue(th); /* Release any queued buffers */ in ___kmp_thread_free()
1219 brel(th, ptr); in ___kmp_thread_free()
1289 __kmp_threads[gtid]->th.th_def_allocator = allocator; in __kmpc_set_default_allocator()
1292 return __kmp_threads[gtid]->th.th_def_allocator; in __kmpc_get_default_allocator()
1306 allocator = __kmp_threads[gtid]->th.th_def_allocator; in __kmpc_alloc()
1589 ptr = this_thr->th.th_free_lists[index].th_free_list_self; in ___kmp_fast_allocate()
1592 this_thr->th.th_free_lists[index].th_free_list_self = *((void **)ptr); in ___kmp_fast_allocate()
1599 ptr = TCR_SYNC_PTR(this_thr->th.th_free_lists[index].th_free_list_sync); in ___kmp_fast_allocate()
1605 &this_thr->th.th_free_lists[index].th_free_list_sync, ptr, nullptr)) { in ___kmp_fast_allocate()
1607 ptr = TCR_SYNC_PTR(this_thr->th.th_free_lists[index].th_free_list_sync); in ___kmp_fast_allocate()
1611 this_thr->th.th_free_lists[index].th_free_list_self = *((void **)ptr); in ___kmp_fast_allocate()
1685 *((void **)ptr) = this_thr->th.th_free_lists[index].th_free_list_self; in ___kmp_fast_free()
1686 this_thr->th.th_free_lists[index].th_free_list_self = ptr; in ___kmp_fast_free()
1688 void *head = this_thr->th.th_free_lists[index].th_free_list_other; in ___kmp_fast_free()
1691 this_thr->th.th_free_lists[index].th_free_list_other = ptr; in ___kmp_fast_free()
1706 this_thr->th.th_free_lists[index].th_free_list_other = ptr; in ___kmp_fast_free()
1727 old_ptr = TCR_PTR(q_th->th.th_free_lists[index].th_free_list_sync); in ___kmp_fast_free()
1733 &q_th->th.th_free_lists[index].th_free_list_sync, old_ptr, head)) { in ___kmp_fast_free()
1735 old_ptr = TCR_PTR(q_th->th.th_free_lists[index].th_free_list_sync); in ___kmp_fast_free()
1740 this_thr->th.th_free_lists[index].th_free_list_other = ptr; in ___kmp_fast_free()
1764 memset(this_thr->th.th_free_lists, 0, NUM_LISTS * sizeof(kmp_free_list_t)); in __kmp_initialize_fast_memory()
1769 void __kmp_free_fast_memory(kmp_info_t *th) { in __kmp_free_fast_memory() argument
1772 thr_data_t *thr = get_thr_data(th); in __kmp_free_fast_memory()
1776 5, ("__kmp_free_fast_memory: Called T#%d\n", __kmp_gtid_from_thread(th))); in __kmp_free_fast_memory()
1778 __kmp_bget_dequeue(th); // Release any queued buffers in __kmp_free_fast_memory()
1795 lst, next, th, __kmp_gtid_from_thread(th))); in __kmp_free_fast_memory()
1806 5, ("__kmp_free_fast_memory: Freed T#%d\n", __kmp_gtid_from_thread(th))); in __kmp_free_fast_memory()