1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 #include <sys/types.h> 10 #include <string.h> 11 #include <sys/queue.h> 12 #include <stdarg.h> 13 #include <errno.h> 14 #include <getopt.h> 15 #include <unistd.h> 16 #include <sched.h> 17 #include <pthread.h> 18 19 #include <rte_common.h> 20 #include <rte_lcore.h> 21 #include <rte_per_lcore.h> 22 #include <rte_timer.h> 23 24 #include "lthread_api.h" 25 #include "lthread_diag_api.h" 26 #include "pthread_shim.h" 27 28 #define DEBUG_APP 0 29 #define HELLOW_WORLD_MAX_LTHREADS 10 30 31 #ifndef __GLIBC__ /* sched_getcpu() is glibc-specific */ 32 #define sched_getcpu() rte_lcore_id() 33 #endif 34 35 __thread int print_count; 36 __thread pthread_mutex_t print_lock; 37 38 __thread pthread_mutex_t exit_lock; 39 __thread pthread_cond_t exit_cond; 40 41 /* 42 * A simple thread that demonstrates use of a mutex, a condition 43 * variable, thread local storage, explicit yield, and thread exit. 44 * 45 * The thread uses a mutex to protect a shared counter which is incremented 46 * and then it waits on condition variable before exiting. 47 * 48 * The thread argument is stored in and retrieved from TLS, using 49 * the pthread key create, get and set specific APIs. 50 * 51 * The thread yields while holding the mutex, to provide opportunity 52 * for other threads to contend. 53 * 54 * All of the pthread API functions used by this thread are actually 55 * resolved to corresponding lthread functions by the pthread shim 56 * implemented in pthread_shim.c 57 */ 58 void *helloworld_pthread(void *arg); 59 void *helloworld_pthread(void *arg) 60 { 61 pthread_key_t key; 62 63 /* create a key for TLS */ 64 pthread_key_create(&key, NULL); 65 66 /* store the arg in TLS */ 67 pthread_setspecific(key, arg); 68 69 /* grab lock and increment shared counter */ 70 pthread_mutex_lock(&print_lock); 71 print_count++; 72 73 /* yield thread to give opportunity for lock contention */ 74 pthread_yield(); 75 76 /* retrieve arg from TLS */ 77 uint64_t thread_no = (uint64_t) pthread_getspecific(key); 78 79 printf("Hello - lcore = %d count = %d thread_no = %d thread_id = %p\n", 80 sched_getcpu(), 81 print_count, 82 (int) thread_no, 83 (void *)pthread_self()); 84 85 /* release the lock */ 86 pthread_mutex_unlock(&print_lock); 87 88 /* 89 * wait on condition variable 90 * before exiting 91 */ 92 pthread_mutex_lock(&exit_lock); 93 pthread_cond_wait(&exit_cond, &exit_lock); 94 pthread_mutex_unlock(&exit_lock); 95 96 /* exit */ 97 pthread_exit((void *) thread_no); 98 } 99 100 101 /* 102 * This is the initial thread 103 * 104 * It demonstrates pthread, mutex and condition variable creation, 105 * broadcast and pthread join APIs. 106 * 107 * This initial thread must always start life as an lthread. 108 * 109 * This thread creates many more threads then waits a short time 110 * before signalling them to exit using a broadcast. 111 * 112 * All of the pthread API functions used by this thread are actually 113 * resolved to corresponding lthread functions by the pthread shim 114 * implemented in pthread_shim.c 115 * 116 * After all threads have finished the lthread scheduler is shutdown 117 * and normal pthread operation is restored 118 */ 119 __thread pthread_t tid[HELLOW_WORLD_MAX_LTHREADS]; 120 121 static void *initial_lthread(void *args __rte_unused) 122 { 123 int lcore = (int) rte_lcore_id(); 124 /* 125 * 126 * We can now enable pthread API override 127 * and start to use the pthread APIs 128 */ 129 pthread_override_set(1); 130 131 uint64_t i; 132 int ret; 133 134 /* initialize mutex for shared counter */ 135 print_count = 0; 136 pthread_mutex_init(&print_lock, NULL); 137 138 /* initialize mutex and condition variable controlling thread exit */ 139 pthread_mutex_init(&exit_lock, NULL); 140 pthread_cond_init(&exit_cond, NULL); 141 142 /* spawn a number of threads */ 143 for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) { 144 145 /* 146 * Not strictly necessary but 147 * for the sake of this example 148 * use an attribute to pass the desired lcore 149 */ 150 pthread_attr_t attr; 151 rte_cpuset_t cpuset; 152 153 CPU_ZERO(&cpuset); 154 CPU_SET(lcore, &cpuset); 155 pthread_attr_init(&attr); 156 pthread_attr_setaffinity_np(&attr, sizeof(rte_cpuset_t), &cpuset); 157 158 /* create the thread */ 159 ret = pthread_create(&tid[i], &attr, 160 helloworld_pthread, (void *) i); 161 if (ret != 0) 162 rte_exit(EXIT_FAILURE, "Cannot create helloworld thread\n"); 163 } 164 165 /* wait for 1s to allow threads 166 * to block on the condition variable 167 * N.B. nanosleep() is resolved to lthread_sleep() 168 * by the shim. 169 */ 170 struct timespec time; 171 172 time.tv_sec = 1; 173 time.tv_nsec = 0; 174 nanosleep(&time, NULL); 175 176 /* wake up all the threads */ 177 pthread_cond_broadcast(&exit_cond); 178 179 /* wait for them to finish */ 180 for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) { 181 182 uint64_t thread_no; 183 184 pthread_join(tid[i], (void *) &thread_no); 185 if (thread_no != i) 186 printf("error on thread exit\n"); 187 } 188 189 pthread_cond_destroy(&exit_cond); 190 pthread_mutex_destroy(&print_lock); 191 pthread_mutex_destroy(&exit_lock); 192 193 /* shutdown the lthread scheduler */ 194 lthread_scheduler_shutdown(rte_lcore_id()); 195 lthread_detach(); 196 return NULL; 197 } 198 199 200 201 /* This thread creates a single initial lthread 202 * and then runs the scheduler 203 * An instance of this thread is created on each thread 204 * in the core mask 205 */ 206 static int 207 lthread_scheduler(void *args __rte_unused) 208 { 209 /* create initial thread */ 210 struct lthread *lt; 211 212 lthread_create(<, -1, initial_lthread, (void *) NULL); 213 214 /* run the lthread scheduler */ 215 lthread_run(); 216 217 /* restore genuine pthread operation */ 218 pthread_override_set(0); 219 return 0; 220 } 221 222 int main(int argc, char **argv) 223 { 224 int num_sched = 0; 225 226 /* basic DPDK initialization is all that is necessary to run lthreads*/ 227 int ret = rte_eal_init(argc, argv); 228 229 if (ret < 0) 230 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 231 232 /* enable timer subsystem */ 233 rte_timer_subsystem_init(); 234 235 #if DEBUG_APP 236 lthread_diagnostic_set_mask(LT_DIAG_ALL); 237 #endif 238 239 /* create a scheduler on every core in the core mask 240 * and launch an initial lthread that will spawn many more. 241 */ 242 unsigned lcore_id; 243 244 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 245 if (rte_lcore_is_enabled(lcore_id)) 246 num_sched++; 247 } 248 249 /* set the number of schedulers, this forces all schedulers synchronize 250 * before entering their main loop 251 */ 252 lthread_num_schedulers_set(num_sched); 253 254 /* launch all threads */ 255 rte_eal_mp_remote_launch(lthread_scheduler, (void *)NULL, CALL_MAIN); 256 257 /* wait for threads to stop */ 258 RTE_LCORE_FOREACH_WORKER(lcore_id) { 259 rte_eal_wait_lcore(lcore_id); 260 } 261 return 0; 262 } 263