1 2 /*- 3 * BSD LICENSE 4 * 5 * Copyright(c) 2015 Intel Corporation. All rights reserved. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * * Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * * Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * * Neither the name of Intel Corporation nor the names of its 19 * contributors may be used to endorse or promote products derived 20 * from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #define _GNU_SOURCE 36 #include <stdio.h> 37 #include <stdlib.h> 38 #include <stdint.h> 39 #include <inttypes.h> 40 #include <sys/types.h> 41 #include <string.h> 42 #include <sys/queue.h> 43 #include <stdarg.h> 44 #include <errno.h> 45 #include <getopt.h> 46 #include <unistd.h> 47 #include <sched.h> 48 #include <pthread.h> 49 50 #include <rte_common.h> 51 #include <rte_lcore.h> 52 #include <rte_per_lcore.h> 53 #include <rte_timer.h> 54 55 #include "lthread_api.h" 56 #include "lthread_diag_api.h" 57 #include "pthread_shim.h" 58 59 #define DEBUG_APP 0 60 #define HELLOW_WORLD_MAX_LTHREADS 10 61 62 __thread int print_count; 63 __thread pthread_mutex_t print_lock; 64 65 __thread pthread_mutex_t exit_lock; 66 __thread pthread_cond_t exit_cond; 67 68 /* 69 * A simple thread that demonstrates use of a mutex, a condition 70 * variable, thread local storage, explicit yield, and thread exit. 71 * 72 * The thread uses a mutex to protect a shared counter which is incremented 73 * and then it waits on condition variable before exiting. 74 * 75 * The thread argument is stored in and retrieved from TLS, using 76 * the pthread key create, get and set specific APIs. 77 * 78 * The thread yields while holding the mutex, to provide opportunity 79 * for other threads to contend. 80 * 81 * All of the pthread API functions used by this thread are actually 82 * resolved to corresponding lthread functions by the pthread shim 83 * implemented in pthread_shim.c 84 */ 85 void *helloworld_pthread(void *arg); 86 void *helloworld_pthread(void *arg) 87 { 88 pthread_key_t key; 89 90 /* create a key for TLS */ 91 pthread_key_create(&key, NULL); 92 93 /* store the arg in TLS */ 94 pthread_setspecific(key, arg); 95 96 /* grab lock and increment shared counter */ 97 pthread_mutex_lock(&print_lock); 98 print_count++; 99 100 /* yield thread to give opportunity for lock contention */ 101 pthread_yield(); 102 103 /* retrieve arg from TLS */ 104 uint64_t thread_no = (uint64_t) pthread_getspecific(key); 105 106 printf("Hello - lcore = %d count = %d thread_no = %d thread_id = %p\n", 107 sched_getcpu(), 108 print_count, 109 (int) thread_no, 110 (void *)pthread_self()); 111 112 /* release the lock */ 113 pthread_mutex_unlock(&print_lock); 114 115 /* 116 * wait on condition variable 117 * before exiting 118 */ 119 pthread_mutex_lock(&exit_lock); 120 pthread_cond_wait(&exit_cond, &exit_lock); 121 pthread_mutex_unlock(&exit_lock); 122 123 /* exit */ 124 pthread_exit((void *) thread_no); 125 } 126 127 128 /* 129 * This is the initial thread 130 * 131 * It demonstrates pthread, mutex and condition variable creation, 132 * broadcast and pthread join APIs. 133 * 134 * This initial thread must always start life as an lthread. 135 * 136 * This thread creates many more threads then waits a short time 137 * before signalling them to exit using a broadcast. 138 * 139 * All of the pthread API functions used by this thread are actually 140 * resolved to corresponding lthread functions by the pthread shim 141 * implemented in pthread_shim.c 142 * 143 * After all threads have finished the lthread scheduler is shutdown 144 * and normal pthread operation is restored 145 */ 146 __thread pthread_t tid[HELLOW_WORLD_MAX_LTHREADS]; 147 148 static void initial_lthread(void *args); 149 static void initial_lthread(void *args __attribute__((unused))) 150 { 151 int lcore = (int) rte_lcore_id(); 152 /* 153 * 154 * We can now enable pthread API override 155 * and start to use the pthread APIs 156 */ 157 pthread_override_set(1); 158 159 uint64_t i; 160 161 /* initialize mutex for shared counter */ 162 print_count = 0; 163 pthread_mutex_init(&print_lock, NULL); 164 165 /* initialize mutex and condition variable controlling thread exit */ 166 pthread_mutex_init(&exit_lock, NULL); 167 pthread_cond_init(&exit_cond, NULL); 168 169 /* spawn a number of threads */ 170 for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) { 171 172 /* 173 * Not strictly necessary but 174 * for the sake of this example 175 * use an attribute to pass the desired lcore 176 */ 177 pthread_attr_t attr; 178 cpu_set_t cpuset; 179 180 CPU_ZERO(&cpuset); 181 CPU_SET(lcore, &cpuset); 182 pthread_attr_init(&attr); 183 pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); 184 185 /* create the thread */ 186 pthread_create(&tid[i], &attr, helloworld_pthread, (void *) i); 187 } 188 189 /* wait for 1s to allow threads 190 * to block on the condition variable 191 * N.B. nanosleep() is resolved to lthread_sleep() 192 * by the shim. 193 */ 194 struct timespec time; 195 196 time.tv_sec = 1; 197 time.tv_nsec = 0; 198 nanosleep(&time, NULL); 199 200 /* wake up all the threads */ 201 pthread_cond_broadcast(&exit_cond); 202 203 /* wait for them to finish */ 204 for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) { 205 206 uint64_t thread_no; 207 208 pthread_join(tid[i], (void *) &thread_no); 209 if (thread_no != i) 210 printf("error on thread exit\n"); 211 } 212 213 pthread_cond_destroy(&exit_cond); 214 pthread_mutex_destroy(&print_lock); 215 pthread_mutex_destroy(&exit_lock); 216 217 /* shutdown the lthread scheduler */ 218 lthread_scheduler_shutdown(rte_lcore_id()); 219 lthread_detach(); 220 } 221 222 223 224 /* This thread creates a single initial lthread 225 * and then runs the scheduler 226 * An instance of this thread is created on each thread 227 * in the core mask 228 */ 229 static int 230 lthread_scheduler(void *args); 231 static int 232 lthread_scheduler(void *args __attribute__((unused))) 233 { 234 /* create initial thread */ 235 struct lthread *lt; 236 237 lthread_create(<, -1, initial_lthread, (void *) NULL); 238 239 /* run the lthread scheduler */ 240 lthread_run(); 241 242 /* restore genuine pthread operation */ 243 pthread_override_set(0); 244 return 0; 245 } 246 247 int main(int argc, char **argv) 248 { 249 int num_sched = 0; 250 251 /* basic DPDK initialization is all that is necessary to run lthreads*/ 252 int ret = rte_eal_init(argc, argv); 253 254 if (ret < 0) 255 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 256 257 /* enable timer subsystem */ 258 rte_timer_subsystem_init(); 259 260 #if DEBUG_APP 261 lthread_diagnostic_set_mask(LT_DIAG_ALL); 262 #endif 263 264 /* create a scheduler on every core in the core mask 265 * and launch an initial lthread that will spawn many more. 266 */ 267 unsigned lcore_id; 268 269 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 270 if (rte_lcore_is_enabled(lcore_id)) 271 num_sched++; 272 } 273 274 /* set the number of schedulers, this forces all schedulers synchronize 275 * before entering their main loop 276 */ 277 lthread_num_schedulers_set(num_sched); 278 279 /* launch all threads */ 280 rte_eal_mp_remote_launch(lthread_scheduler, (void *)NULL, CALL_MASTER); 281 282 /* wait for threads to stop */ 283 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 284 rte_eal_wait_lcore(lcore_id); 285 } 286 return 0; 287 } 288