1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2010-2015 Intel Corporation 4 * Copyright (c) 2007,2008 Kip Macy [email protected] 5 * All rights reserved. 6 * Derived from FreeBSD's bufring.h 7 * Used as BSD-3 Licensed with permission from Kip Macy. 8 */ 9 10 #include <stdio.h> 11 #include <stdarg.h> 12 #include <string.h> 13 #include <stdint.h> 14 #include <inttypes.h> 15 #include <errno.h> 16 #include <sys/queue.h> 17 18 #include <rte_common.h> 19 #include <rte_log.h> 20 #include <rte_memory.h> 21 #include <rte_memzone.h> 22 #include <rte_malloc.h> 23 #include <rte_launch.h> 24 #include <rte_eal.h> 25 #include <rte_eal_memconfig.h> 26 #include <rte_atomic.h> 27 #include <rte_per_lcore.h> 28 #include <rte_lcore.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_errno.h> 31 #include <rte_string_fns.h> 32 #include <rte_spinlock.h> 33 34 #include "rte_ring.h" 35 36 TAILQ_HEAD(rte_ring_list, rte_tailq_entry); 37 38 static struct rte_tailq_elem rte_ring_tailq = { 39 .name = RTE_TAILQ_RING_NAME, 40 }; 41 EAL_REGISTER_TAILQ(rte_ring_tailq) 42 43 /* true if x is a power of 2 */ 44 #define POWEROF2(x) ((((x)-1) & (x)) == 0) 45 46 /* return the size of memory occupied by a ring */ 47 ssize_t 48 rte_ring_get_memsize(unsigned count) 49 { 50 ssize_t sz; 51 52 /* count must be a power of 2 */ 53 if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK )) { 54 RTE_LOG(ERR, RING, 55 "Requested size is invalid, must be power of 2, and " 56 "do not exceed the size limit %u\n", RTE_RING_SZ_MASK); 57 return -EINVAL; 58 } 59 60 sz = sizeof(struct rte_ring) + count * sizeof(void *); 61 sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE); 62 return sz; 63 } 64 65 int 66 rte_ring_init(struct rte_ring *r, const char *name, unsigned count, 67 unsigned flags) 68 { 69 int ret; 70 71 /* compilation-time checks */ 72 RTE_BUILD_BUG_ON((sizeof(struct rte_ring) & 73 RTE_CACHE_LINE_MASK) != 0); 74 RTE_BUILD_BUG_ON((offsetof(struct rte_ring, cons) & 75 RTE_CACHE_LINE_MASK) != 0); 76 RTE_BUILD_BUG_ON((offsetof(struct rte_ring, prod) & 77 RTE_CACHE_LINE_MASK) != 0); 78 79 /* init the ring structure */ 80 memset(r, 0, sizeof(*r)); 81 ret = snprintf(r->name, sizeof(r->name), "%s", name); 82 if (ret < 0 || ret >= (int)sizeof(r->name)) 83 return -ENAMETOOLONG; 84 r->flags = flags; 85 r->prod.single = (flags & RING_F_SP_ENQ) ? __IS_SP : __IS_MP; 86 r->cons.single = (flags & RING_F_SC_DEQ) ? __IS_SC : __IS_MC; 87 88 if (flags & RING_F_EXACT_SZ) { 89 r->size = rte_align32pow2(count + 1); 90 r->mask = r->size - 1; 91 r->capacity = count; 92 } else { 93 if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK)) { 94 RTE_LOG(ERR, RING, 95 "Requested size is invalid, must be power of 2, and not exceed the size limit %u\n", 96 RTE_RING_SZ_MASK); 97 return -EINVAL; 98 } 99 r->size = count; 100 r->mask = count - 1; 101 r->capacity = r->mask; 102 } 103 r->prod.head = r->cons.head = 0; 104 r->prod.tail = r->cons.tail = 0; 105 106 return 0; 107 } 108 109 /* create the ring */ 110 struct rte_ring * 111 rte_ring_create(const char *name, unsigned count, int socket_id, 112 unsigned flags) 113 { 114 char mz_name[RTE_MEMZONE_NAMESIZE]; 115 struct rte_ring *r; 116 struct rte_tailq_entry *te; 117 const struct rte_memzone *mz; 118 ssize_t ring_size; 119 int mz_flags = 0; 120 struct rte_ring_list* ring_list = NULL; 121 const unsigned int requested_count = count; 122 int ret; 123 124 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 125 126 /* for an exact size ring, round up from count to a power of two */ 127 if (flags & RING_F_EXACT_SZ) 128 count = rte_align32pow2(count + 1); 129 130 ring_size = rte_ring_get_memsize(count); 131 if (ring_size < 0) { 132 rte_errno = ring_size; 133 return NULL; 134 } 135 136 ret = snprintf(mz_name, sizeof(mz_name), "%s%s", 137 RTE_RING_MZ_PREFIX, name); 138 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 139 rte_errno = ENAMETOOLONG; 140 return NULL; 141 } 142 143 te = rte_zmalloc("RING_TAILQ_ENTRY", sizeof(*te), 0); 144 if (te == NULL) { 145 RTE_LOG(ERR, RING, "Cannot reserve memory for tailq\n"); 146 rte_errno = ENOMEM; 147 return NULL; 148 } 149 150 rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); 151 152 /* reserve a memory zone for this ring. If we can't get rte_config or 153 * we are secondary process, the memzone_reserve function will set 154 * rte_errno for us appropriately - hence no check in this this function */ 155 mz = rte_memzone_reserve_aligned(mz_name, ring_size, socket_id, 156 mz_flags, __alignof__(*r)); 157 if (mz != NULL) { 158 r = mz->addr; 159 /* no need to check return value here, we already checked the 160 * arguments above */ 161 rte_ring_init(r, name, requested_count, flags); 162 163 te->data = (void *) r; 164 r->memzone = mz; 165 166 TAILQ_INSERT_TAIL(ring_list, te, next); 167 } else { 168 r = NULL; 169 RTE_LOG(ERR, RING, "Cannot reserve memory\n"); 170 rte_free(te); 171 } 172 rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); 173 174 return r; 175 } 176 177 /* free the ring */ 178 void 179 rte_ring_free(struct rte_ring *r) 180 { 181 struct rte_ring_list *ring_list = NULL; 182 struct rte_tailq_entry *te; 183 184 if (r == NULL) 185 return; 186 187 /* 188 * Ring was not created with rte_ring_create, 189 * therefore, there is no memzone to free. 190 */ 191 if (r->memzone == NULL) { 192 RTE_LOG(ERR, RING, 193 "Cannot free ring, not created with rte_ring_create()\n"); 194 return; 195 } 196 197 if (rte_memzone_free(r->memzone) != 0) { 198 RTE_LOG(ERR, RING, "Cannot free memory\n"); 199 return; 200 } 201 202 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 203 rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); 204 205 /* find out tailq entry */ 206 TAILQ_FOREACH(te, ring_list, next) { 207 if (te->data == (void *) r) 208 break; 209 } 210 211 if (te == NULL) { 212 rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); 213 return; 214 } 215 216 TAILQ_REMOVE(ring_list, te, next); 217 218 rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); 219 220 rte_free(te); 221 } 222 223 /* dump the status of the ring on the console */ 224 void 225 rte_ring_dump(FILE *f, const struct rte_ring *r) 226 { 227 fprintf(f, "ring <%s>@%p\n", r->name, r); 228 fprintf(f, " flags=%x\n", r->flags); 229 fprintf(f, " size=%"PRIu32"\n", r->size); 230 fprintf(f, " capacity=%"PRIu32"\n", r->capacity); 231 fprintf(f, " ct=%"PRIu32"\n", r->cons.tail); 232 fprintf(f, " ch=%"PRIu32"\n", r->cons.head); 233 fprintf(f, " pt=%"PRIu32"\n", r->prod.tail); 234 fprintf(f, " ph=%"PRIu32"\n", r->prod.head); 235 fprintf(f, " used=%u\n", rte_ring_count(r)); 236 fprintf(f, " avail=%u\n", rte_ring_free_count(r)); 237 } 238 239 /* dump the status of all rings on the console */ 240 void 241 rte_ring_list_dump(FILE *f) 242 { 243 const struct rte_tailq_entry *te; 244 struct rte_ring_list *ring_list; 245 246 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 247 248 rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK); 249 250 TAILQ_FOREACH(te, ring_list, next) { 251 rte_ring_dump(f, (struct rte_ring *) te->data); 252 } 253 254 rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK); 255 } 256 257 /* search a ring from its name */ 258 struct rte_ring * 259 rte_ring_lookup(const char *name) 260 { 261 struct rte_tailq_entry *te; 262 struct rte_ring *r = NULL; 263 struct rte_ring_list *ring_list; 264 265 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 266 267 rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK); 268 269 TAILQ_FOREACH(te, ring_list, next) { 270 r = (struct rte_ring *) te->data; 271 if (strncmp(name, r->name, RTE_RING_NAMESIZE) == 0) 272 break; 273 } 274 275 rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK); 276 277 if (te == NULL) { 278 rte_errno = ENOENT; 279 return NULL; 280 } 281 282 return r; 283 } 284