1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2015 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <string.h> 37 #include <stdint.h> 38 #include <limits.h> 39 #include <inttypes.h> 40 #include <unistd.h> 41 #include <pthread.h> 42 #include <fcntl.h> 43 #include <sys/time.h> 44 #include <sys/mman.h> 45 #include <sched.h> 46 47 #include <rte_malloc.h> 48 #include <rte_log.h> 49 #include <rte_ring.h> 50 #include <rte_atomic_64.h> 51 52 #include "lthread_tls.h" 53 #include "lthread_queue.h" 54 #include "lthread_objcache.h" 55 #include "lthread_sched.h" 56 57 static struct rte_ring *key_pool; 58 static uint64_t key_pool_init; 59 60 /* needed to cause section start and end to be defined */ 61 RTE_DEFINE_PER_LTHREAD(void *, dummy); 62 63 static struct lthread_key key_table[LTHREAD_MAX_KEYS]; 64 65 RTE_INIT(thread_tls_ctor) 66 { 67 key_pool = NULL; 68 key_pool_init = 0; 69 } 70 71 /* 72 * Initialize a pool of keys 73 * These are unique tokens that can be obtained by threads 74 * calling lthread_key_create() 75 */ 76 void _lthread_key_pool_init(void) 77 { 78 static struct rte_ring *pool; 79 struct lthread_key *new_key; 80 char name[MAX_LTHREAD_NAME_SIZE]; 81 82 bzero(key_table, sizeof(key_table)); 83 84 /* only one lcore should do this */ 85 if (rte_atomic64_cmpset(&key_pool_init, 0, 1)) { 86 87 snprintf(name, 88 MAX_LTHREAD_NAME_SIZE, 89 "lthread_key_pool_%d", 90 getpid()); 91 92 pool = rte_ring_create(name, 93 LTHREAD_MAX_KEYS, 0, 0); 94 RTE_ASSERT(pool); 95 96 int i; 97 98 for (i = 1; i < LTHREAD_MAX_KEYS; i++) { 99 new_key = &key_table[i]; 100 rte_ring_mp_enqueue((struct rte_ring *)pool, 101 (void *)new_key); 102 } 103 key_pool = pool; 104 } 105 /* other lcores wait here till done */ 106 while (key_pool == NULL) { 107 rte_compiler_barrier(); 108 sched_yield(); 109 }; 110 } 111 112 /* 113 * Create a key 114 * this means getting a key from the the pool 115 */ 116 int lthread_key_create(unsigned int *key, tls_destructor_func destructor) 117 { 118 if (key == NULL) 119 return POSIX_ERRNO(EINVAL); 120 121 struct lthread_key *new_key; 122 123 if (rte_ring_mc_dequeue((struct rte_ring *)key_pool, (void **)&new_key) 124 == 0) { 125 new_key->destructor = destructor; 126 *key = (new_key - key_table); 127 128 return 0; 129 } 130 return POSIX_ERRNO(EAGAIN); 131 } 132 133 134 /* 135 * Delete a key 136 */ 137 int lthread_key_delete(unsigned int k) 138 { 139 struct lthread_key *key; 140 141 key = (struct lthread_key *) &key_table[k]; 142 143 if (k > LTHREAD_MAX_KEYS) 144 return POSIX_ERRNO(EINVAL); 145 146 key->destructor = NULL; 147 rte_ring_mp_enqueue((struct rte_ring *)key_pool, 148 (void *)key); 149 return 0; 150 } 151 152 153 154 /* 155 * Break association for all keys in use by this thread 156 * invoke the destructor if available. 157 * Since a destructor can create keys we could enter an infinite loop 158 * therefore we give up after LTHREAD_DESTRUCTOR_ITERATIONS 159 * the behavior is modelled on pthread 160 */ 161 void _lthread_tls_destroy(struct lthread *lt) 162 { 163 int i, k; 164 int nb_keys; 165 void *data; 166 167 for (i = 0; i < LTHREAD_DESTRUCTOR_ITERATIONS; i++) { 168 169 for (k = 1; k < LTHREAD_MAX_KEYS; k++) { 170 171 /* no keys in use ? */ 172 nb_keys = lt->tls->nb_keys_inuse; 173 if (nb_keys == 0) 174 return; 175 176 /* this key not in use ? */ 177 if (lt->tls->data[k] == NULL) 178 continue; 179 180 /* remove this key */ 181 data = lt->tls->data[k]; 182 lt->tls->data[k] = NULL; 183 lt->tls->nb_keys_inuse = nb_keys-1; 184 185 /* invoke destructor */ 186 if (key_table[k].destructor != NULL) 187 key_table[k].destructor(data); 188 } 189 } 190 } 191 192 /* 193 * Return the pointer associated with a key 194 * If the key is no longer valid return NULL 195 */ 196 void 197 *lthread_getspecific(unsigned int k) 198 { 199 void *res = NULL; 200 201 if (k < LTHREAD_MAX_KEYS) 202 res = THIS_LTHREAD->tls->data[k]; 203 204 return res; 205 } 206 207 /* 208 * Set a value against a key 209 * If the key is no longer valid return an error 210 * when storing value 211 */ 212 int lthread_setspecific(unsigned int k, const void *data) 213 { 214 if (k >= LTHREAD_MAX_KEYS) 215 return POSIX_ERRNO(EINVAL); 216 217 int n = THIS_LTHREAD->tls->nb_keys_inuse; 218 219 /* discard const qualifier */ 220 char *p = (char *) (uintptr_t) data; 221 222 223 if (data != NULL) { 224 if (THIS_LTHREAD->tls->data[k] == NULL) 225 THIS_LTHREAD->tls->nb_keys_inuse = n+1; 226 } 227 228 THIS_LTHREAD->tls->data[k] = (void *) p; 229 return 0; 230 } 231 232 /* 233 * Allocate data for TLS cache 234 */ 235 void _lthread_tls_alloc(struct lthread *lt) 236 { 237 struct lthread_tls *tls; 238 239 tls = _lthread_objcache_alloc((THIS_SCHED)->tls_cache); 240 241 RTE_ASSERT(tls != NULL); 242 243 tls->root_sched = (THIS_SCHED); 244 lt->tls = tls; 245 246 /* allocate data for TLS varaiables using RTE_PER_LTHREAD macros */ 247 if (sizeof(void *) < (uint64_t)RTE_PER_LTHREAD_SECTION_SIZE) { 248 lt->per_lthread_data = 249 _lthread_objcache_alloc((THIS_SCHED)->per_lthread_cache); 250 } 251 } 252