Lines Matching refs:thread_data
28 kmp_thread_data_t *thread_data);
43 kmp_thread_data_t *thread_data, in __kmp_trace_task_stack() argument
45 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; in __kmp_trace_task_stack()
97 kmp_thread_data_t *thread_data) { in __kmp_init_task_stack() argument
98 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; in __kmp_init_task_stack()
118 kmp_thread_data_t *thread_data) { in __kmp_free_task_stack() argument
119 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; in __kmp_free_task_stack()
149 kmp_thread_data_t *thread_data = in __kmp_push_task_stack() local
151 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; in __kmp_push_task_stack()
208 kmp_thread_data_t *thread_data = in __kmp_pop_task_stack() local
210 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; in __kmp_pop_task_stack()
297 kmp_thread_data_t *thread_data) { in __kmp_realloc_task_deque() argument
298 kmp_int32 size = TASK_DEQUE_SIZE(thread_data->td); in __kmp_realloc_task_deque()
299 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == size); in __kmp_realloc_task_deque()
304 __kmp_gtid_from_thread(thread), size, new_size, thread_data)); in __kmp_realloc_task_deque()
310 for (i = thread_data->td.td_deque_head, j = 0; j < size; in __kmp_realloc_task_deque()
311 i = (i + 1) & TASK_DEQUE_MASK(thread_data->td), j++) in __kmp_realloc_task_deque()
312 new_deque[j] = thread_data->td.td_deque[i]; in __kmp_realloc_task_deque()
314 __kmp_free(thread_data->td.td_deque); in __kmp_realloc_task_deque()
316 thread_data->td.td_deque_head = 0; in __kmp_realloc_task_deque()
317 thread_data->td.td_deque_tail = size; in __kmp_realloc_task_deque()
318 thread_data->td.td_deque = new_deque; in __kmp_realloc_task_deque()
319 thread_data->td.td_deque_size = new_size; in __kmp_realloc_task_deque()
324 kmp_thread_data_t *thread_data = &l->td; in __kmp_alloc_task_pri_list() local
325 __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_alloc_task_pri_list()
326 thread_data->td.td_deque_last_stolen = -1; in __kmp_alloc_task_pri_list()
329 __kmp_get_gtid(), INITIAL_TASK_DEQUE_SIZE, thread_data)); in __kmp_alloc_task_pri_list()
330 thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate( in __kmp_alloc_task_pri_list()
332 thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE; in __kmp_alloc_task_pri_list()
343 kmp_thread_data_t *thread_data; in __kmp_get_priority_deque_data() local
347 thread_data = &lst->td; in __kmp_get_priority_deque_data()
352 thread_data = &list->td; in __kmp_get_priority_deque_data()
366 thread_data = &list->td; in __kmp_get_priority_deque_data()
372 thread_data = &next_queue->td; in __kmp_get_priority_deque_data()
376 thread_data = &list->td; in __kmp_get_priority_deque_data()
382 return thread_data; in __kmp_get_priority_deque_data()
390 kmp_thread_data_t *thread_data = NULL; in __kmp_push_priority_task() local
402 thread_data = &list->td; in __kmp_push_priority_task()
408 thread_data = __kmp_get_priority_deque_data(task_team, pri); in __kmp_push_priority_task()
414 thread_data = &lst->td; in __kmp_push_priority_task()
417 thread_data = __kmp_get_priority_deque_data(task_team, pri); in __kmp_push_priority_task()
421 KMP_DEBUG_ASSERT(thread_data); in __kmp_push_priority_task()
423 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_push_priority_task()
425 if (TCR_4(thread_data->td.td_deque_ntasks) >= in __kmp_push_priority_task()
426 TASK_DEQUE_SIZE(thread_data->td)) { in __kmp_push_priority_task()
430 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_push_priority_task()
437 __kmp_realloc_task_deque(thread, thread_data); in __kmp_push_priority_task()
440 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) < in __kmp_push_priority_task()
441 TASK_DEQUE_SIZE(thread_data->td)); in __kmp_push_priority_task()
443 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata; in __kmp_push_priority_task()
445 thread_data->td.td_deque_tail = in __kmp_push_priority_task()
446 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); in __kmp_push_priority_task()
447 TCW_4(thread_data->td.td_deque_ntasks, in __kmp_push_priority_task()
448 TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count in __kmp_push_priority_task()
453 gtid, taskdata, thread_data->td.td_deque_ntasks, in __kmp_push_priority_task()
454 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); in __kmp_push_priority_task()
455 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_push_priority_task()
479 kmp_thread_data_t *thread_data; in __kmp_push_task() local
519 thread_data = &task_team->tt.tt_threads_data[tid]; in __kmp_push_task()
524 if (UNLIKELY(thread_data->td.td_deque == NULL)) { in __kmp_push_task()
525 __kmp_alloc_task_deque(thread, thread_data); in __kmp_push_task()
530 if (TCR_4(thread_data->td.td_deque_ntasks) >= in __kmp_push_task()
531 TASK_DEQUE_SIZE(thread_data->td)) { in __kmp_push_task()
540 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_push_task()
542 if (TCR_4(thread_data->td.td_deque_ntasks) >= in __kmp_push_task()
543 TASK_DEQUE_SIZE(thread_data->td)) { in __kmp_push_task()
545 __kmp_realloc_task_deque(thread, thread_data); in __kmp_push_task()
551 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_push_task()
553 if (TCR_4(thread_data->td.td_deque_ntasks) >= in __kmp_push_task()
554 TASK_DEQUE_SIZE(thread_data->td)) { in __kmp_push_task()
558 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_push_task()
565 __kmp_realloc_task_deque(thread, thread_data); in __kmp_push_task()
570 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) < in __kmp_push_task()
571 TASK_DEQUE_SIZE(thread_data->td)); in __kmp_push_task()
573 thread_data->td.td_deque[thread_data->td.td_deque_tail] = in __kmp_push_task()
576 thread_data->td.td_deque_tail = in __kmp_push_task()
577 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); in __kmp_push_task()
578 TCW_4(thread_data->td.td_deque_ntasks, in __kmp_push_task()
579 TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count in __kmp_push_task()
584 gtid, taskdata, thread_data->td.td_deque_ntasks, in __kmp_push_task()
585 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); in __kmp_push_task()
587 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_push_task()
1425 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; in __kmp_task_alloc() local
1427 if (thread_data->td.td_deque == NULL) { in __kmp_task_alloc()
1428 __kmp_alloc_task_deque(thread, thread_data); in __kmp_task_alloc()
2851 kmp_thread_data_t *thread_data; in __kmp_get_priority_task() local
2874 thread_data = &list->td; in __kmp_get_priority_task()
2875 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_get_priority_task()
2876 deque_ntasks = thread_data->td.td_deque_ntasks; in __kmp_get_priority_task()
2878 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_get_priority_task()
2880 __kmp_get_gtid(), thread_data)); in __kmp_get_priority_task()
2885 int target = thread_data->td.td_deque_head; in __kmp_get_priority_task()
2887 taskdata = thread_data->td.td_deque[target]; in __kmp_get_priority_task()
2890 thread_data->td.td_deque_head = in __kmp_get_priority_task()
2891 (target + 1) & TASK_DEQUE_MASK(thread_data->td); in __kmp_get_priority_task()
2895 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_get_priority_task()
2898 gtid, thread_data, task_team, deque_ntasks, target, in __kmp_get_priority_task()
2899 thread_data->td.td_deque_tail)); in __kmp_get_priority_task()
2907 target = (target + 1) & TASK_DEQUE_MASK(thread_data->td); in __kmp_get_priority_task()
2908 taskdata = thread_data->td.td_deque[target]; in __kmp_get_priority_task()
2917 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_get_priority_task()
2921 gtid, thread_data, task_team, deque_ntasks, in __kmp_get_priority_task()
2922 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); in __kmp_get_priority_task()
2929 target = (target + 1) & TASK_DEQUE_MASK(thread_data->td); in __kmp_get_priority_task()
2930 thread_data->td.td_deque[prev] = thread_data->td.td_deque[target]; in __kmp_get_priority_task()
2934 thread_data->td.td_deque_tail == in __kmp_get_priority_task()
2935 (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(thread_data->td))); in __kmp_get_priority_task()
2936 thread_data->td.td_deque_tail = target; // tail -= 1 (wrapped)) in __kmp_get_priority_task()
2938 thread_data->td.td_deque_ntasks = deque_ntasks - 1; in __kmp_get_priority_task()
2939 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_get_priority_task()
2950 kmp_thread_data_t *thread_data; in __kmp_remove_my_task() local
2957 thread_data = &task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)]; in __kmp_remove_my_task()
2960 gtid, thread_data->td.td_deque_ntasks, in __kmp_remove_my_task()
2961 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); in __kmp_remove_my_task()
2963 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) { in __kmp_remove_my_task()
2967 gtid, thread_data->td.td_deque_ntasks, in __kmp_remove_my_task()
2968 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); in __kmp_remove_my_task()
2972 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_remove_my_task()
2974 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) { in __kmp_remove_my_task()
2975 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_remove_my_task()
2979 gtid, thread_data->td.td_deque_ntasks, in __kmp_remove_my_task()
2980 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); in __kmp_remove_my_task()
2984 tail = (thread_data->td.td_deque_tail - 1) & in __kmp_remove_my_task()
2985 TASK_DEQUE_MASK(thread_data->td); // Wrap index. in __kmp_remove_my_task()
2986 taskdata = thread_data->td.td_deque[tail]; in __kmp_remove_my_task()
2991 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_remove_my_task()
2995 gtid, thread_data->td.td_deque_ntasks, in __kmp_remove_my_task()
2996 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); in __kmp_remove_my_task()
3000 thread_data->td.td_deque_tail = tail; in __kmp_remove_my_task()
3001 TCW_4(thread_data->td.td_deque_ntasks, thread_data->td.td_deque_ntasks - 1); in __kmp_remove_my_task()
3003 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_remove_my_task()
3007 gtid, taskdata, thread_data->td.td_deque_ntasks, in __kmp_remove_my_task()
3008 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); in __kmp_remove_my_task()
3553 kmp_thread_data_t *thread_data) { in __kmp_alloc_task_deque() argument
3554 __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_alloc_task_deque()
3555 KMP_DEBUG_ASSERT(thread_data->td.td_deque == NULL); in __kmp_alloc_task_deque()
3558 thread_data->td.td_deque_last_stolen = -1; in __kmp_alloc_task_deque()
3560 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == 0); in __kmp_alloc_task_deque()
3561 KMP_DEBUG_ASSERT(thread_data->td.td_deque_head == 0); in __kmp_alloc_task_deque()
3562 KMP_DEBUG_ASSERT(thread_data->td.td_deque_tail == 0); in __kmp_alloc_task_deque()
3567 __kmp_gtid_from_thread(thread), INITIAL_TASK_DEQUE_SIZE, thread_data)); in __kmp_alloc_task_deque()
3571 thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate( in __kmp_alloc_task_deque()
3573 thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE; in __kmp_alloc_task_deque()
3579 static void __kmp_free_task_deque(kmp_thread_data_t *thread_data) { in __kmp_free_task_deque() argument
3580 if (thread_data->td.td_deque != NULL) { in __kmp_free_task_deque()
3581 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_free_task_deque()
3582 TCW_4(thread_data->td.td_deque_ntasks, 0); in __kmp_free_task_deque()
3583 __kmp_free(thread_data->td.td_deque); in __kmp_free_task_deque()
3584 thread_data->td.td_deque = NULL; in __kmp_free_task_deque()
3585 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_free_task_deque()
3590 if (thread_data->td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY) { in __kmp_free_task_deque()
3591 __kmp_free_task_stack(__kmp_thread_from_gtid(gtid), thread_data); in __kmp_free_task_deque()
3653 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; in __kmp_realloc_task_threads_data() local
3654 __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data); in __kmp_realloc_task_threads_data()
3672 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; in __kmp_realloc_task_threads_data() local
3673 __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data); in __kmp_realloc_task_threads_data()
3685 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; in __kmp_realloc_task_threads_data() local
3686 thread_data->td.td_thr = team->t.t_threads[i]; in __kmp_realloc_task_threads_data()
3688 if (thread_data->td.td_deque_last_stolen >= nthreads) { in __kmp_realloc_task_threads_data()
3692 thread_data->td.td_deque_last_stolen = -1; in __kmp_realloc_task_threads_data()
3983 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[j]; in __kmp_task_team_setup() local
3984 if (thread_data->td.td_deque == NULL) { in __kmp_task_team_setup()
3985 __kmp_alloc_task_deque(__kmp_hidden_helper_threads[j], thread_data); in __kmp_task_team_setup()
4114 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; in __kmp_give_task() local
4116 if (thread_data->td.td_deque == NULL) { in __kmp_give_task()
4125 if (TCR_4(thread_data->td.td_deque_ntasks) >= in __kmp_give_task()
4126 TASK_DEQUE_SIZE(thread_data->td)) { in __kmp_give_task()
4134 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass) in __kmp_give_task()
4137 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_give_task()
4138 if (TCR_4(thread_data->td.td_deque_ntasks) >= in __kmp_give_task()
4139 TASK_DEQUE_SIZE(thread_data->td)) { in __kmp_give_task()
4141 __kmp_realloc_task_deque(thread, thread_data); in __kmp_give_task()
4146 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_give_task()
4148 if (TCR_4(thread_data->td.td_deque_ntasks) >= in __kmp_give_task()
4149 TASK_DEQUE_SIZE(thread_data->td)) { in __kmp_give_task()
4156 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass) in __kmp_give_task()
4159 __kmp_realloc_task_deque(thread, thread_data); in __kmp_give_task()
4165 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata; in __kmp_give_task()
4167 thread_data->td.td_deque_tail = in __kmp_give_task()
4168 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); in __kmp_give_task()
4169 TCW_4(thread_data->td.td_deque_ntasks, in __kmp_give_task()
4170 TCR_4(thread_data->td.td_deque_ntasks) + 1); in __kmp_give_task()
4177 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); in __kmp_give_task()