1 2 /*- 3 * BSD LICENSE 4 * 5 * Copyright(c) 2015 Intel Corporation. All rights reserved. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * * Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * * Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * * Neither the name of Intel Corporation nor the names of its 19 * contributors may be used to endorse or promote products derived 20 * from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #define _GNU_SOURCE 36 #include <stdio.h> 37 #include <stdlib.h> 38 #include <stdint.h> 39 #include <inttypes.h> 40 #include <sys/types.h> 41 #include <string.h> 42 #include <sys/queue.h> 43 #include <stdarg.h> 44 #include <errno.h> 45 #include <getopt.h> 46 #include <unistd.h> 47 #include <sched.h> 48 #include <pthread.h> 49 50 #include <rte_common.h> 51 #include <rte_lcore.h> 52 #include <rte_per_lcore.h> 53 #include <rte_timer.h> 54 55 #include "lthread_api.h" 56 #include "lthread_diag_api.h" 57 #include "pthread_shim.h" 58 59 #define DEBUG_APP 0 60 #define HELLOW_WORLD_MAX_LTHREADS 10 61 62 #ifndef __GLIBC__ /* sched_getcpu() is glibc-specific */ 63 #define sched_getcpu() rte_lcore_id() 64 #endif 65 66 __thread int print_count; 67 __thread pthread_mutex_t print_lock; 68 69 __thread pthread_mutex_t exit_lock; 70 __thread pthread_cond_t exit_cond; 71 72 /* 73 * A simple thread that demonstrates use of a mutex, a condition 74 * variable, thread local storage, explicit yield, and thread exit. 75 * 76 * The thread uses a mutex to protect a shared counter which is incremented 77 * and then it waits on condition variable before exiting. 78 * 79 * The thread argument is stored in and retrieved from TLS, using 80 * the pthread key create, get and set specific APIs. 81 * 82 * The thread yields while holding the mutex, to provide opportunity 83 * for other threads to contend. 84 * 85 * All of the pthread API functions used by this thread are actually 86 * resolved to corresponding lthread functions by the pthread shim 87 * implemented in pthread_shim.c 88 */ 89 void *helloworld_pthread(void *arg); 90 void *helloworld_pthread(void *arg) 91 { 92 pthread_key_t key; 93 94 /* create a key for TLS */ 95 pthread_key_create(&key, NULL); 96 97 /* store the arg in TLS */ 98 pthread_setspecific(key, arg); 99 100 /* grab lock and increment shared counter */ 101 pthread_mutex_lock(&print_lock); 102 print_count++; 103 104 /* yield thread to give opportunity for lock contention */ 105 pthread_yield(); 106 107 /* retrieve arg from TLS */ 108 uint64_t thread_no = (uint64_t) pthread_getspecific(key); 109 110 printf("Hello - lcore = %d count = %d thread_no = %d thread_id = %p\n", 111 sched_getcpu(), 112 print_count, 113 (int) thread_no, 114 (void *)pthread_self()); 115 116 /* release the lock */ 117 pthread_mutex_unlock(&print_lock); 118 119 /* 120 * wait on condition variable 121 * before exiting 122 */ 123 pthread_mutex_lock(&exit_lock); 124 pthread_cond_wait(&exit_cond, &exit_lock); 125 pthread_mutex_unlock(&exit_lock); 126 127 /* exit */ 128 pthread_exit((void *) thread_no); 129 } 130 131 132 /* 133 * This is the initial thread 134 * 135 * It demonstrates pthread, mutex and condition variable creation, 136 * broadcast and pthread join APIs. 137 * 138 * This initial thread must always start life as an lthread. 139 * 140 * This thread creates many more threads then waits a short time 141 * before signalling them to exit using a broadcast. 142 * 143 * All of the pthread API functions used by this thread are actually 144 * resolved to corresponding lthread functions by the pthread shim 145 * implemented in pthread_shim.c 146 * 147 * After all threads have finished the lthread scheduler is shutdown 148 * and normal pthread operation is restored 149 */ 150 __thread pthread_t tid[HELLOW_WORLD_MAX_LTHREADS]; 151 152 static void *initial_lthread(void *args __attribute__((unused))) 153 { 154 int lcore = (int) rte_lcore_id(); 155 /* 156 * 157 * We can now enable pthread API override 158 * and start to use the pthread APIs 159 */ 160 pthread_override_set(1); 161 162 uint64_t i; 163 int ret; 164 165 /* initialize mutex for shared counter */ 166 print_count = 0; 167 pthread_mutex_init(&print_lock, NULL); 168 169 /* initialize mutex and condition variable controlling thread exit */ 170 pthread_mutex_init(&exit_lock, NULL); 171 pthread_cond_init(&exit_cond, NULL); 172 173 /* spawn a number of threads */ 174 for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) { 175 176 /* 177 * Not strictly necessary but 178 * for the sake of this example 179 * use an attribute to pass the desired lcore 180 */ 181 pthread_attr_t attr; 182 rte_cpuset_t cpuset; 183 184 CPU_ZERO(&cpuset); 185 CPU_SET(lcore, &cpuset); 186 pthread_attr_init(&attr); 187 pthread_attr_setaffinity_np(&attr, sizeof(rte_cpuset_t), &cpuset); 188 189 /* create the thread */ 190 ret = pthread_create(&tid[i], &attr, 191 helloworld_pthread, (void *) i); 192 if (ret != 0) 193 rte_exit(EXIT_FAILURE, "Cannot create helloworld thread\n"); 194 } 195 196 /* wait for 1s to allow threads 197 * to block on the condition variable 198 * N.B. nanosleep() is resolved to lthread_sleep() 199 * by the shim. 200 */ 201 struct timespec time; 202 203 time.tv_sec = 1; 204 time.tv_nsec = 0; 205 nanosleep(&time, NULL); 206 207 /* wake up all the threads */ 208 pthread_cond_broadcast(&exit_cond); 209 210 /* wait for them to finish */ 211 for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) { 212 213 uint64_t thread_no; 214 215 pthread_join(tid[i], (void *) &thread_no); 216 if (thread_no != i) 217 printf("error on thread exit\n"); 218 } 219 220 pthread_cond_destroy(&exit_cond); 221 pthread_mutex_destroy(&print_lock); 222 pthread_mutex_destroy(&exit_lock); 223 224 /* shutdown the lthread scheduler */ 225 lthread_scheduler_shutdown(rte_lcore_id()); 226 lthread_detach(); 227 return NULL; 228 } 229 230 231 232 /* This thread creates a single initial lthread 233 * and then runs the scheduler 234 * An instance of this thread is created on each thread 235 * in the core mask 236 */ 237 static int 238 lthread_scheduler(void *args __attribute__((unused))) 239 { 240 /* create initial thread */ 241 struct lthread *lt; 242 243 lthread_create(<, -1, initial_lthread, (void *) NULL); 244 245 /* run the lthread scheduler */ 246 lthread_run(); 247 248 /* restore genuine pthread operation */ 249 pthread_override_set(0); 250 return 0; 251 } 252 253 int main(int argc, char **argv) 254 { 255 int num_sched = 0; 256 257 /* basic DPDK initialization is all that is necessary to run lthreads*/ 258 int ret = rte_eal_init(argc, argv); 259 260 if (ret < 0) 261 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 262 263 /* enable timer subsystem */ 264 rte_timer_subsystem_init(); 265 266 #if DEBUG_APP 267 lthread_diagnostic_set_mask(LT_DIAG_ALL); 268 #endif 269 270 /* create a scheduler on every core in the core mask 271 * and launch an initial lthread that will spawn many more. 272 */ 273 unsigned lcore_id; 274 275 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 276 if (rte_lcore_is_enabled(lcore_id)) 277 num_sched++; 278 } 279 280 /* set the number of schedulers, this forces all schedulers synchronize 281 * before entering their main loop 282 */ 283 lthread_num_schedulers_set(num_sched); 284 285 /* launch all threads */ 286 rte_eal_mp_remote_launch(lthread_scheduler, (void *)NULL, CALL_MASTER); 287 288 /* wait for threads to stop */ 289 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 290 rte_eal_wait_lcore(lcore_id); 291 } 292 return 0; 293 } 294