1 // RUN: %libomp-compile && env LIBOMP_NUM_HIDDEN_HELPER_THREADS=0 OMP_PROC_BIND=close OMP_PLACES=cores KMP_AFFINITY=verbose %libomp-run 8 1 4 2 // REQUIRES: linux 3 // 4 // This test pthread_creates 8 root threads before any OpenMP 5 // runtime entry is ever called. We have all the root threads 6 // register with the runtime by calling omp_set_num_threads(), 7 // but this does not initialize their affinity. The fourth root thread 8 // then calls a parallel region and we make sure its affinity 9 // is correct. We also make sure all the other root threads are 10 // free-floating since they have not called into a parallel region. 11 12 #define _GNU_SOURCE 13 #include <stdio.h> 14 #include <stdlib.h> 15 #include <omp.h> 16 #include <pthread.h> 17 #include <unistd.h> 18 #include <assert.h> 19 #include <sys/types.h> 20 #include <sys/syscall.h> 21 #include "libomp_test_affinity.h" 22 23 volatile int entry_flag = 0; 24 volatile int flag = 0; 25 volatile int num_roots_arrived = 0; 26 int num_roots; 27 int spawner = 0; 28 pthread_mutex_t lock; 29 int register_workers = 0; // boolean 30 affinity_mask_t *full_mask; 31 32 int __kmpc_global_thread_num(void*); 33 34 int get_os_thread_id() { 35 return (int)syscall(SYS_gettid); 36 } 37 38 int place_and_affinity_match() { 39 int i, max_cpu; 40 char buf[512]; 41 affinity_mask_t *mask = affinity_mask_alloc(); 42 int place = omp_get_place_num(); 43 int num_procs = omp_get_place_num_procs(place); 44 int *ids = (int*)malloc(sizeof(int) * num_procs); 45 omp_get_place_proc_ids(place, ids); 46 get_thread_affinity(mask); 47 affinity_mask_snprintf(buf, sizeof(buf), mask); 48 printf("Primary Thread Place: %d\n", place); 49 printf("Primary Thread mask: %s\n", buf); 50 51 for (i = 0; i < num_procs; ++i) { 52 int cpu = ids[i]; 53 if (!affinity_mask_isset(mask, cpu)) 54 return 0; 55 } 56 57 max_cpu = AFFINITY_MAX_CPUS; 58 for (i = 0; i < max_cpu; ++i) { 59 int cpu = i; 60 if (affinity_mask_isset(mask, cpu)) { 61 int j, found = 0; 62 for (j = 0; j < num_procs; ++j) { 63 if (ids[j] == cpu) { 64 found = 1; 65 break; 66 } 67 } 68 if (!found) 69 return 0; 70 } 71 } 72 73 affinity_mask_free(mask); 74 free(ids); 75 return 1; 76 } 77 78 void* thread_func(void *arg) { 79 int place, nplaces; 80 int root_id = *((int*)arg); 81 int pid = getpid(); 82 int tid = get_os_thread_id(); 83 84 // Order how the root threads are assigned a gtid in the runtime 85 // i.e., root_id = gtid 86 while (1) { 87 int v = entry_flag; 88 if (v == root_id) 89 break; 90 } 91 92 // If main root thread 93 if (root_id == spawner) { 94 printf("Initial application thread (pid=%d, tid=%d, spawner=%d) reached thread_func (will call OpenMP)\n", pid, tid, spawner); 95 omp_set_num_threads(4); 96 #pragma omp atomic 97 entry_flag++; 98 // Wait for the workers to signal their arrival before #pragma omp parallel 99 while (num_roots_arrived < num_roots - 1) {} 100 // This will trigger the output for KMP_AFFINITY in this case 101 #pragma omp parallel 102 { 103 int gtid = __kmpc_global_thread_num(NULL); 104 #pragma omp single 105 { 106 printf("Exactly %d threads in the #pragma omp parallel\n", 107 omp_get_num_threads()); 108 } 109 #pragma omp critical 110 { 111 printf("OpenMP thread %d: gtid=%d\n", omp_get_thread_num(), gtid); 112 } 113 } 114 flag = 1; 115 if (!place_and_affinity_match()) { 116 fprintf(stderr, "error: place and affinity mask do not match for primary thread\n"); 117 exit (EXIT_FAILURE); 118 } 119 120 } else { // If worker root thread 121 // Worker root threads, register with OpenMP through omp_set_num_threads() 122 // if designated to, signal their arrival and then wait for the main root 123 // thread to signal them to exit. 124 printf("New root pthread (pid=%d, tid=%d) reached thread_func\n", pid, tid); 125 if (register_workers) 126 omp_set_num_threads(4); 127 #pragma omp atomic 128 entry_flag++; 129 130 pthread_mutex_lock(&lock); 131 num_roots_arrived++; 132 pthread_mutex_unlock(&lock); 133 while (flag == 0) {} 134 135 // Main check whether root threads' mask is equal to the 136 // initial affinity mask 137 affinity_mask_t *mask = affinity_mask_alloc(); 138 get_thread_affinity(mask); 139 if (!affinity_mask_equal(mask, full_mask)) { 140 char buf[1024]; 141 printf("root thread %d mask: ", root_id); 142 affinity_mask_snprintf(buf, sizeof(buf), mask); 143 printf("initial affinity mask: %s\n", buf); 144 fprintf(stderr, "error: root thread %d affinity mask not equal" 145 " to initial full mask\n", root_id); 146 affinity_mask_free(mask); 147 exit(EXIT_FAILURE); 148 } 149 affinity_mask_free(mask); 150 } 151 return NULL; 152 } 153 154 int main(int argc, char** argv) { 155 int i; 156 if (argc != 3 && argc != 4) { 157 fprintf(stderr, "usage: %s <num_roots> <register_workers_bool> [<spawn_root_number>]\n", argv[0]); 158 exit(EXIT_FAILURE); 159 } 160 161 // Initialize pthread mutex 162 pthread_mutex_init(&lock, NULL); 163 164 // Get initial full mask 165 full_mask = affinity_mask_alloc(); 166 get_thread_affinity(full_mask); 167 168 // Get the number of root pthreads to create and allocate resources for them 169 num_roots = atoi(argv[1]); 170 pthread_t *roots = (pthread_t*)malloc(sizeof(pthread_t) * num_roots); 171 int *root_ids = (int*)malloc(sizeof(int) * num_roots); 172 173 // Get the flag indicating whether to have root pthreads call omp_set_num_threads() or not 174 register_workers = atoi(argv[2]); 175 176 if (argc == 4) 177 spawner = atoi(argv[3]); 178 179 // Spawn worker root threads 180 for (i = 1; i < num_roots; ++i) { 181 *(root_ids + i) = i; 182 pthread_create(roots + i, NULL, thread_func, root_ids + i); 183 } 184 // Have main root thread (root 0) go into thread_func 185 *root_ids = 0; 186 thread_func(root_ids); 187 188 // Cleanup all resources 189 for (i = 1; i < num_roots; ++i) { 190 void *status; 191 pthread_join(roots[i], &status); 192 } 193 free(roots); 194 free(root_ids); 195 pthread_mutex_destroy(&lock); 196 return EXIT_SUCCESS; 197 } 198