1 2 /*- 3 * BSD LICENSE 4 * 5 * Copyright(c) 2015 Intel Corporation. All rights reserved. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * * Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * * Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * * Neither the name of Intel Corporation nor the names of its 19 * contributors may be used to endorse or promote products derived 20 * from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #define _GNU_SOURCE 36 #include <stdio.h> 37 #include <stdlib.h> 38 #include <stdint.h> 39 #include <inttypes.h> 40 #include <sys/types.h> 41 #include <string.h> 42 #include <sys/queue.h> 43 #include <stdarg.h> 44 #include <errno.h> 45 #include <getopt.h> 46 #include <unistd.h> 47 #include <sched.h> 48 #include <pthread.h> 49 50 #include <rte_common.h> 51 #include <rte_lcore.h> 52 #include <rte_per_lcore.h> 53 #include <rte_timer.h> 54 55 #include "lthread_api.h" 56 #include "lthread_diag_api.h" 57 #include "pthread_shim.h" 58 59 #define DEBUG_APP 0 60 #define HELLOW_WORLD_MAX_LTHREADS 10 61 62 #ifndef __GLIBC__ /* sched_getcpu() is glibc-specific */ 63 #define sched_getcpu() rte_lcore_id() 64 #endif 65 66 __thread int print_count; 67 __thread pthread_mutex_t print_lock; 68 69 __thread pthread_mutex_t exit_lock; 70 __thread pthread_cond_t exit_cond; 71 72 /* 73 * A simple thread that demonstrates use of a mutex, a condition 74 * variable, thread local storage, explicit yield, and thread exit. 75 * 76 * The thread uses a mutex to protect a shared counter which is incremented 77 * and then it waits on condition variable before exiting. 78 * 79 * The thread argument is stored in and retrieved from TLS, using 80 * the pthread key create, get and set specific APIs. 81 * 82 * The thread yields while holding the mutex, to provide opportunity 83 * for other threads to contend. 84 * 85 * All of the pthread API functions used by this thread are actually 86 * resolved to corresponding lthread functions by the pthread shim 87 * implemented in pthread_shim.c 88 */ 89 void *helloworld_pthread(void *arg); 90 void *helloworld_pthread(void *arg) 91 { 92 pthread_key_t key; 93 94 /* create a key for TLS */ 95 pthread_key_create(&key, NULL); 96 97 /* store the arg in TLS */ 98 pthread_setspecific(key, arg); 99 100 /* grab lock and increment shared counter */ 101 pthread_mutex_lock(&print_lock); 102 print_count++; 103 104 /* yield thread to give opportunity for lock contention */ 105 pthread_yield(); 106 107 /* retrieve arg from TLS */ 108 uint64_t thread_no = (uint64_t) pthread_getspecific(key); 109 110 printf("Hello - lcore = %d count = %d thread_no = %d thread_id = %p\n", 111 sched_getcpu(), 112 print_count, 113 (int) thread_no, 114 (void *)pthread_self()); 115 116 /* release the lock */ 117 pthread_mutex_unlock(&print_lock); 118 119 /* 120 * wait on condition variable 121 * before exiting 122 */ 123 pthread_mutex_lock(&exit_lock); 124 pthread_cond_wait(&exit_cond, &exit_lock); 125 pthread_mutex_unlock(&exit_lock); 126 127 /* exit */ 128 pthread_exit((void *) thread_no); 129 } 130 131 132 /* 133 * This is the initial thread 134 * 135 * It demonstrates pthread, mutex and condition variable creation, 136 * broadcast and pthread join APIs. 137 * 138 * This initial thread must always start life as an lthread. 139 * 140 * This thread creates many more threads then waits a short time 141 * before signalling them to exit using a broadcast. 142 * 143 * All of the pthread API functions used by this thread are actually 144 * resolved to corresponding lthread functions by the pthread shim 145 * implemented in pthread_shim.c 146 * 147 * After all threads have finished the lthread scheduler is shutdown 148 * and normal pthread operation is restored 149 */ 150 __thread pthread_t tid[HELLOW_WORLD_MAX_LTHREADS]; 151 152 static void initial_lthread(void *args); 153 static void initial_lthread(void *args __attribute__((unused))) 154 { 155 int lcore = (int) rte_lcore_id(); 156 /* 157 * 158 * We can now enable pthread API override 159 * and start to use the pthread APIs 160 */ 161 pthread_override_set(1); 162 163 uint64_t i; 164 int ret; 165 166 /* initialize mutex for shared counter */ 167 print_count = 0; 168 pthread_mutex_init(&print_lock, NULL); 169 170 /* initialize mutex and condition variable controlling thread exit */ 171 pthread_mutex_init(&exit_lock, NULL); 172 pthread_cond_init(&exit_cond, NULL); 173 174 /* spawn a number of threads */ 175 for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) { 176 177 /* 178 * Not strictly necessary but 179 * for the sake of this example 180 * use an attribute to pass the desired lcore 181 */ 182 pthread_attr_t attr; 183 rte_cpuset_t cpuset; 184 185 CPU_ZERO(&cpuset); 186 CPU_SET(lcore, &cpuset); 187 pthread_attr_init(&attr); 188 pthread_attr_setaffinity_np(&attr, sizeof(rte_cpuset_t), &cpuset); 189 190 /* create the thread */ 191 ret = pthread_create(&tid[i], &attr, 192 helloworld_pthread, (void *) i); 193 if (ret != 0) 194 rte_exit(EXIT_FAILURE, "Cannot create helloworld thread\n"); 195 } 196 197 /* wait for 1s to allow threads 198 * to block on the condition variable 199 * N.B. nanosleep() is resolved to lthread_sleep() 200 * by the shim. 201 */ 202 struct timespec time; 203 204 time.tv_sec = 1; 205 time.tv_nsec = 0; 206 nanosleep(&time, NULL); 207 208 /* wake up all the threads */ 209 pthread_cond_broadcast(&exit_cond); 210 211 /* wait for them to finish */ 212 for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) { 213 214 uint64_t thread_no; 215 216 pthread_join(tid[i], (void *) &thread_no); 217 if (thread_no != i) 218 printf("error on thread exit\n"); 219 } 220 221 pthread_cond_destroy(&exit_cond); 222 pthread_mutex_destroy(&print_lock); 223 pthread_mutex_destroy(&exit_lock); 224 225 /* shutdown the lthread scheduler */ 226 lthread_scheduler_shutdown(rte_lcore_id()); 227 lthread_detach(); 228 } 229 230 231 232 /* This thread creates a single initial lthread 233 * and then runs the scheduler 234 * An instance of this thread is created on each thread 235 * in the core mask 236 */ 237 static int 238 lthread_scheduler(void *args); 239 static int 240 lthread_scheduler(void *args __attribute__((unused))) 241 { 242 /* create initial thread */ 243 struct lthread *lt; 244 245 lthread_create(<, -1, initial_lthread, (void *) NULL); 246 247 /* run the lthread scheduler */ 248 lthread_run(); 249 250 /* restore genuine pthread operation */ 251 pthread_override_set(0); 252 return 0; 253 } 254 255 int main(int argc, char **argv) 256 { 257 int num_sched = 0; 258 259 /* basic DPDK initialization is all that is necessary to run lthreads*/ 260 int ret = rte_eal_init(argc, argv); 261 262 if (ret < 0) 263 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 264 265 /* enable timer subsystem */ 266 rte_timer_subsystem_init(); 267 268 #if DEBUG_APP 269 lthread_diagnostic_set_mask(LT_DIAG_ALL); 270 #endif 271 272 /* create a scheduler on every core in the core mask 273 * and launch an initial lthread that will spawn many more. 274 */ 275 unsigned lcore_id; 276 277 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 278 if (rte_lcore_is_enabled(lcore_id)) 279 num_sched++; 280 } 281 282 /* set the number of schedulers, this forces all schedulers synchronize 283 * before entering their main loop 284 */ 285 lthread_num_schedulers_set(num_sched); 286 287 /* launch all threads */ 288 rte_eal_mp_remote_launch(lthread_scheduler, (void *)NULL, CALL_MASTER); 289 290 /* wait for threads to stop */ 291 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 292 rte_eal_wait_lcore(lcore_id); 293 } 294 return 0; 295 } 296