1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation. 3 * Copyright(c) 2016 6WIND S.A. 4 */ 5 6 #include <stdbool.h> 7 #include <stdio.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <stdarg.h> 11 #include <unistd.h> 12 #include <inttypes.h> 13 #include <errno.h> 14 #include <sys/queue.h> 15 16 #include <rte_common.h> 17 #include <rte_log.h> 18 #include <rte_debug.h> 19 #include <rte_memory.h> 20 #include <rte_memzone.h> 21 #include <rte_malloc.h> 22 #include <rte_atomic.h> 23 #include <rte_launch.h> 24 #include <rte_eal.h> 25 #include <rte_eal_memconfig.h> 26 #include <rte_per_lcore.h> 27 #include <rte_lcore.h> 28 #include <rte_branch_prediction.h> 29 #include <rte_errno.h> 30 #include <rte_string_fns.h> 31 #include <rte_spinlock.h> 32 #include <rte_tailq.h> 33 #include <rte_eal_paging.h> 34 #include <rte_telemetry.h> 35 36 #include "rte_mempool.h" 37 #include "rte_mempool_trace.h" 38 39 TAILQ_HEAD(rte_mempool_list, rte_tailq_entry); 40 41 static struct rte_tailq_elem rte_mempool_tailq = { 42 .name = "RTE_MEMPOOL", 43 }; 44 EAL_REGISTER_TAILQ(rte_mempool_tailq) 45 46 TAILQ_HEAD(mempool_callback_list, rte_tailq_entry); 47 48 static struct rte_tailq_elem callback_tailq = { 49 .name = "RTE_MEMPOOL_CALLBACK", 50 }; 51 EAL_REGISTER_TAILQ(callback_tailq) 52 53 /* Invoke all registered mempool event callbacks. */ 54 static void 55 mempool_event_callback_invoke(enum rte_mempool_event event, 56 struct rte_mempool *mp); 57 58 #define CACHE_FLUSHTHRESH_MULTIPLIER 1.5 59 #define CALC_CACHE_FLUSHTHRESH(c) \ 60 ((typeof(c))((c) * CACHE_FLUSHTHRESH_MULTIPLIER)) 61 62 #if defined(RTE_ARCH_X86) 63 /* 64 * return the greatest common divisor between a and b (fast algorithm) 65 * 66 */ 67 static unsigned get_gcd(unsigned a, unsigned b) 68 { 69 unsigned c; 70 71 if (0 == a) 72 return b; 73 if (0 == b) 74 return a; 75 76 if (a < b) { 77 c = a; 78 a = b; 79 b = c; 80 } 81 82 while (b != 0) { 83 c = a % b; 84 a = b; 85 b = c; 86 } 87 88 return a; 89 } 90 91 /* 92 * Depending on memory configuration on x86 arch, objects addresses are spread 93 * between channels and ranks in RAM: the pool allocator will add 94 * padding between objects. This function return the new size of the 95 * object. 96 */ 97 static unsigned int 98 arch_mem_object_align(unsigned int obj_size) 99 { 100 unsigned nrank, nchan; 101 unsigned new_obj_size; 102 103 /* get number of channels */ 104 nchan = rte_memory_get_nchannel(); 105 if (nchan == 0) 106 nchan = 4; 107 108 nrank = rte_memory_get_nrank(); 109 if (nrank == 0) 110 nrank = 1; 111 112 /* process new object size */ 113 new_obj_size = (obj_size + RTE_MEMPOOL_ALIGN_MASK) / RTE_MEMPOOL_ALIGN; 114 while (get_gcd(new_obj_size, nrank * nchan) != 1) 115 new_obj_size++; 116 return new_obj_size * RTE_MEMPOOL_ALIGN; 117 } 118 #else 119 static unsigned int 120 arch_mem_object_align(unsigned int obj_size) 121 { 122 return obj_size; 123 } 124 #endif 125 126 struct pagesz_walk_arg { 127 int socket_id; 128 size_t min; 129 }; 130 131 static int 132 find_min_pagesz(const struct rte_memseg_list *msl, void *arg) 133 { 134 struct pagesz_walk_arg *wa = arg; 135 bool valid; 136 137 /* 138 * we need to only look at page sizes available for a particular socket 139 * ID. so, we either need an exact match on socket ID (can match both 140 * native and external memory), or, if SOCKET_ID_ANY was specified as a 141 * socket ID argument, we must only look at native memory and ignore any 142 * page sizes associated with external memory. 143 */ 144 valid = msl->socket_id == wa->socket_id; 145 valid |= wa->socket_id == SOCKET_ID_ANY && msl->external == 0; 146 147 if (valid && msl->page_sz < wa->min) 148 wa->min = msl->page_sz; 149 150 return 0; 151 } 152 153 static size_t 154 get_min_page_size(int socket_id) 155 { 156 struct pagesz_walk_arg wa; 157 158 wa.min = SIZE_MAX; 159 wa.socket_id = socket_id; 160 161 rte_memseg_list_walk(find_min_pagesz, &wa); 162 163 return wa.min == SIZE_MAX ? (size_t) rte_mem_page_size() : wa.min; 164 } 165 166 167 static void 168 mempool_add_elem(struct rte_mempool *mp, __rte_unused void *opaque, 169 void *obj, rte_iova_t iova) 170 { 171 struct rte_mempool_objhdr *hdr; 172 struct rte_mempool_objtlr *tlr __rte_unused; 173 174 /* set mempool ptr in header */ 175 hdr = RTE_PTR_SUB(obj, sizeof(*hdr)); 176 hdr->mp = mp; 177 hdr->iova = iova; 178 STAILQ_INSERT_TAIL(&mp->elt_list, hdr, next); 179 mp->populated_size++; 180 181 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 182 hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE2; 183 tlr = rte_mempool_get_trailer(obj); 184 tlr->cookie = RTE_MEMPOOL_TRAILER_COOKIE; 185 #endif 186 } 187 188 /* call obj_cb() for each mempool element */ 189 uint32_t 190 rte_mempool_obj_iter(struct rte_mempool *mp, 191 rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg) 192 { 193 struct rte_mempool_objhdr *hdr; 194 void *obj; 195 unsigned n = 0; 196 197 STAILQ_FOREACH(hdr, &mp->elt_list, next) { 198 obj = (char *)hdr + sizeof(*hdr); 199 obj_cb(mp, obj_cb_arg, obj, n); 200 n++; 201 } 202 203 return n; 204 } 205 206 /* call mem_cb() for each mempool memory chunk */ 207 uint32_t 208 rte_mempool_mem_iter(struct rte_mempool *mp, 209 rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg) 210 { 211 struct rte_mempool_memhdr *hdr; 212 unsigned n = 0; 213 214 STAILQ_FOREACH(hdr, &mp->mem_list, next) { 215 mem_cb(mp, mem_cb_arg, hdr, n); 216 n++; 217 } 218 219 return n; 220 } 221 222 /* get the header, trailer and total size of a mempool element. */ 223 uint32_t 224 rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, 225 struct rte_mempool_objsz *sz) 226 { 227 struct rte_mempool_objsz lsz; 228 229 sz = (sz != NULL) ? sz : &lsz; 230 231 sz->header_size = sizeof(struct rte_mempool_objhdr); 232 if ((flags & RTE_MEMPOOL_F_NO_CACHE_ALIGN) == 0) 233 sz->header_size = RTE_ALIGN_CEIL(sz->header_size, 234 RTE_MEMPOOL_ALIGN); 235 236 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 237 sz->trailer_size = sizeof(struct rte_mempool_objtlr); 238 #else 239 sz->trailer_size = 0; 240 #endif 241 242 /* element size is 8 bytes-aligned at least */ 243 sz->elt_size = RTE_ALIGN_CEIL(elt_size, sizeof(uint64_t)); 244 245 /* expand trailer to next cache line */ 246 if ((flags & RTE_MEMPOOL_F_NO_CACHE_ALIGN) == 0) { 247 sz->total_size = sz->header_size + sz->elt_size + 248 sz->trailer_size; 249 sz->trailer_size += ((RTE_MEMPOOL_ALIGN - 250 (sz->total_size & RTE_MEMPOOL_ALIGN_MASK)) & 251 RTE_MEMPOOL_ALIGN_MASK); 252 } 253 254 /* 255 * increase trailer to add padding between objects in order to 256 * spread them across memory channels/ranks 257 */ 258 if ((flags & RTE_MEMPOOL_F_NO_SPREAD) == 0) { 259 unsigned new_size; 260 new_size = arch_mem_object_align 261 (sz->header_size + sz->elt_size + sz->trailer_size); 262 sz->trailer_size = new_size - sz->header_size - sz->elt_size; 263 } 264 265 /* this is the size of an object, including header and trailer */ 266 sz->total_size = sz->header_size + sz->elt_size + sz->trailer_size; 267 268 return sz->total_size; 269 } 270 271 /* free a memchunk allocated with rte_memzone_reserve() */ 272 static void 273 rte_mempool_memchunk_mz_free(__rte_unused struct rte_mempool_memhdr *memhdr, 274 void *opaque) 275 { 276 const struct rte_memzone *mz = opaque; 277 rte_memzone_free(mz); 278 } 279 280 /* Free memory chunks used by a mempool. Objects must be in pool */ 281 static void 282 rte_mempool_free_memchunks(struct rte_mempool *mp) 283 { 284 struct rte_mempool_memhdr *memhdr; 285 void *elt; 286 287 while (!STAILQ_EMPTY(&mp->elt_list)) { 288 rte_mempool_ops_dequeue_bulk(mp, &elt, 1); 289 (void)elt; 290 STAILQ_REMOVE_HEAD(&mp->elt_list, next); 291 mp->populated_size--; 292 } 293 294 while (!STAILQ_EMPTY(&mp->mem_list)) { 295 memhdr = STAILQ_FIRST(&mp->mem_list); 296 STAILQ_REMOVE_HEAD(&mp->mem_list, next); 297 if (memhdr->free_cb != NULL) 298 memhdr->free_cb(memhdr, memhdr->opaque); 299 rte_free(memhdr); 300 mp->nb_mem_chunks--; 301 } 302 } 303 304 static int 305 mempool_ops_alloc_once(struct rte_mempool *mp) 306 { 307 int ret; 308 309 /* create the internal ring if not already done */ 310 if ((mp->flags & RTE_MEMPOOL_F_POOL_CREATED) == 0) { 311 ret = rte_mempool_ops_alloc(mp); 312 if (ret != 0) 313 return ret; 314 mp->flags |= RTE_MEMPOOL_F_POOL_CREATED; 315 } 316 return 0; 317 } 318 319 /* Add objects in the pool, using a physically contiguous memory 320 * zone. Return the number of objects added, or a negative value 321 * on error. 322 */ 323 int 324 rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, 325 rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, 326 void *opaque) 327 { 328 unsigned i = 0; 329 size_t off; 330 struct rte_mempool_memhdr *memhdr; 331 int ret; 332 333 ret = mempool_ops_alloc_once(mp); 334 if (ret != 0) 335 return ret; 336 337 /* mempool is already populated */ 338 if (mp->populated_size >= mp->size) 339 return -ENOSPC; 340 341 memhdr = rte_zmalloc("MEMPOOL_MEMHDR", sizeof(*memhdr), 0); 342 if (memhdr == NULL) 343 return -ENOMEM; 344 345 memhdr->mp = mp; 346 memhdr->addr = vaddr; 347 memhdr->iova = iova; 348 memhdr->len = len; 349 memhdr->free_cb = free_cb; 350 memhdr->opaque = opaque; 351 352 if (mp->flags & RTE_MEMPOOL_F_NO_CACHE_ALIGN) 353 off = RTE_PTR_ALIGN_CEIL(vaddr, 8) - vaddr; 354 else 355 off = RTE_PTR_ALIGN_CEIL(vaddr, RTE_MEMPOOL_ALIGN) - vaddr; 356 357 if (off > len) { 358 ret = 0; 359 goto fail; 360 } 361 362 i = rte_mempool_ops_populate(mp, mp->size - mp->populated_size, 363 (char *)vaddr + off, 364 (iova == RTE_BAD_IOVA) ? RTE_BAD_IOVA : (iova + off), 365 len - off, mempool_add_elem, NULL); 366 367 /* not enough room to store one object */ 368 if (i == 0) { 369 ret = 0; 370 goto fail; 371 } 372 373 STAILQ_INSERT_TAIL(&mp->mem_list, memhdr, next); 374 mp->nb_mem_chunks++; 375 376 /* Check if at least some objects in the pool are now usable for IO. */ 377 if (!(mp->flags & RTE_MEMPOOL_F_NO_IOVA_CONTIG) && iova != RTE_BAD_IOVA) 378 mp->flags &= ~RTE_MEMPOOL_F_NON_IO; 379 380 /* Report the mempool as ready only when fully populated. */ 381 if (mp->populated_size >= mp->size) 382 mempool_event_callback_invoke(RTE_MEMPOOL_EVENT_READY, mp); 383 384 rte_mempool_trace_populate_iova(mp, vaddr, iova, len, free_cb, opaque); 385 return i; 386 387 fail: 388 rte_free(memhdr); 389 return ret; 390 } 391 392 static rte_iova_t 393 get_iova(void *addr) 394 { 395 struct rte_memseg *ms; 396 397 /* try registered memory first */ 398 ms = rte_mem_virt2memseg(addr, NULL); 399 if (ms == NULL || ms->iova == RTE_BAD_IOVA) 400 /* fall back to actual physical address */ 401 return rte_mem_virt2iova(addr); 402 return ms->iova + RTE_PTR_DIFF(addr, ms->addr); 403 } 404 405 /* Populate the mempool with a virtual area. Return the number of 406 * objects added, or a negative value on error. 407 */ 408 int 409 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, 410 size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, 411 void *opaque) 412 { 413 rte_iova_t iova; 414 size_t off, phys_len; 415 int ret, cnt = 0; 416 417 if (mp->flags & RTE_MEMPOOL_F_NO_IOVA_CONTIG) 418 return rte_mempool_populate_iova(mp, addr, RTE_BAD_IOVA, 419 len, free_cb, opaque); 420 421 for (off = 0; off < len && 422 mp->populated_size < mp->size; off += phys_len) { 423 424 iova = get_iova(addr + off); 425 426 /* populate with the largest group of contiguous pages */ 427 for (phys_len = RTE_MIN( 428 (size_t)(RTE_PTR_ALIGN_CEIL(addr + off + 1, pg_sz) - 429 (addr + off)), 430 len - off); 431 off + phys_len < len; 432 phys_len = RTE_MIN(phys_len + pg_sz, len - off)) { 433 rte_iova_t iova_tmp; 434 435 iova_tmp = get_iova(addr + off + phys_len); 436 437 if (iova_tmp == RTE_BAD_IOVA || 438 iova_tmp != iova + phys_len) 439 break; 440 } 441 442 ret = rte_mempool_populate_iova(mp, addr + off, iova, 443 phys_len, free_cb, opaque); 444 if (ret == 0) 445 continue; 446 if (ret < 0) 447 goto fail; 448 /* no need to call the free callback for next chunks */ 449 free_cb = NULL; 450 cnt += ret; 451 } 452 453 rte_mempool_trace_populate_virt(mp, addr, len, pg_sz, free_cb, opaque); 454 return cnt; 455 456 fail: 457 rte_mempool_free_memchunks(mp); 458 return ret; 459 } 460 461 /* Get the minimal page size used in a mempool before populating it. */ 462 int 463 rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz) 464 { 465 bool need_iova_contig_obj; 466 bool alloc_in_ext_mem; 467 int ret; 468 469 /* check if we can retrieve a valid socket ID */ 470 ret = rte_malloc_heap_socket_is_external(mp->socket_id); 471 if (ret < 0) 472 return -EINVAL; 473 alloc_in_ext_mem = (ret == 1); 474 need_iova_contig_obj = !(mp->flags & RTE_MEMPOOL_F_NO_IOVA_CONTIG); 475 476 if (!need_iova_contig_obj) 477 *pg_sz = 0; 478 else if (rte_eal_has_hugepages() || alloc_in_ext_mem) 479 *pg_sz = get_min_page_size(mp->socket_id); 480 else 481 *pg_sz = rte_mem_page_size(); 482 483 rte_mempool_trace_get_page_size(mp, *pg_sz); 484 return 0; 485 } 486 487 /* Default function to populate the mempool: allocate memory in memzones, 488 * and populate them. Return the number of objects added, or a negative 489 * value on error. 490 */ 491 int 492 rte_mempool_populate_default(struct rte_mempool *mp) 493 { 494 unsigned int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; 495 char mz_name[RTE_MEMZONE_NAMESIZE]; 496 const struct rte_memzone *mz; 497 ssize_t mem_size; 498 size_t align, pg_sz, pg_shift = 0; 499 rte_iova_t iova; 500 unsigned mz_id, n; 501 int ret; 502 bool need_iova_contig_obj; 503 size_t max_alloc_size = SIZE_MAX; 504 505 ret = mempool_ops_alloc_once(mp); 506 if (ret != 0) 507 return ret; 508 509 /* mempool must not be populated */ 510 if (mp->nb_mem_chunks != 0) 511 return -EEXIST; 512 513 /* 514 * the following section calculates page shift and page size values. 515 * 516 * these values impact the result of calc_mem_size operation, which 517 * returns the amount of memory that should be allocated to store the 518 * desired number of objects. when not zero, it allocates more memory 519 * for the padding between objects, to ensure that an object does not 520 * cross a page boundary. in other words, page size/shift are to be set 521 * to zero if mempool elements won't care about page boundaries. 522 * there are several considerations for page size and page shift here. 523 * 524 * if we don't need our mempools to have physically contiguous objects, 525 * then just set page shift and page size to 0, because the user has 526 * indicated that there's no need to care about anything. 527 * 528 * if we do need contiguous objects (if a mempool driver has its 529 * own calc_size() method returning min_chunk_size = mem_size), 530 * there is also an option to reserve the entire mempool memory 531 * as one contiguous block of memory. 532 * 533 * if we require contiguous objects, but not necessarily the entire 534 * mempool reserved space to be contiguous, pg_sz will be != 0, 535 * and the default ops->populate() will take care of not placing 536 * objects across pages. 537 * 538 * if our IO addresses are physical, we may get memory from bigger 539 * pages, or we might get memory from smaller pages, and how much of it 540 * we require depends on whether we want bigger or smaller pages. 541 * However, requesting each and every memory size is too much work, so 542 * what we'll do instead is walk through the page sizes available, pick 543 * the smallest one and set up page shift to match that one. We will be 544 * wasting some space this way, but it's much nicer than looping around 545 * trying to reserve each and every page size. 546 * 547 * If we fail to get enough contiguous memory, then we'll go and 548 * reserve space in smaller chunks. 549 */ 550 551 need_iova_contig_obj = !(mp->flags & RTE_MEMPOOL_F_NO_IOVA_CONTIG); 552 ret = rte_mempool_get_page_size(mp, &pg_sz); 553 if (ret < 0) 554 return ret; 555 556 if (pg_sz != 0) 557 pg_shift = rte_bsf32(pg_sz); 558 559 for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) { 560 size_t min_chunk_size; 561 562 mem_size = rte_mempool_ops_calc_mem_size( 563 mp, n, pg_shift, &min_chunk_size, &align); 564 565 if (mem_size < 0) { 566 ret = mem_size; 567 goto fail; 568 } 569 570 ret = snprintf(mz_name, sizeof(mz_name), 571 RTE_MEMPOOL_MZ_FORMAT "_%d", mp->name, mz_id); 572 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 573 ret = -ENAMETOOLONG; 574 goto fail; 575 } 576 577 /* if we're trying to reserve contiguous memory, add appropriate 578 * memzone flag. 579 */ 580 if (min_chunk_size == (size_t)mem_size) 581 mz_flags |= RTE_MEMZONE_IOVA_CONTIG; 582 583 /* Allocate a memzone, retrying with a smaller area on ENOMEM */ 584 do { 585 mz = rte_memzone_reserve_aligned(mz_name, 586 RTE_MIN((size_t)mem_size, max_alloc_size), 587 mp->socket_id, mz_flags, align); 588 589 if (mz != NULL || rte_errno != ENOMEM) 590 break; 591 592 max_alloc_size = RTE_MIN(max_alloc_size, 593 (size_t)mem_size) / 2; 594 } while (mz == NULL && max_alloc_size >= min_chunk_size); 595 596 if (mz == NULL) { 597 ret = -rte_errno; 598 goto fail; 599 } 600 601 if (need_iova_contig_obj) 602 iova = mz->iova; 603 else 604 iova = RTE_BAD_IOVA; 605 606 if (pg_sz == 0 || (mz_flags & RTE_MEMZONE_IOVA_CONTIG)) 607 ret = rte_mempool_populate_iova(mp, mz->addr, 608 iova, mz->len, 609 rte_mempool_memchunk_mz_free, 610 (void *)(uintptr_t)mz); 611 else 612 ret = rte_mempool_populate_virt(mp, mz->addr, 613 mz->len, pg_sz, 614 rte_mempool_memchunk_mz_free, 615 (void *)(uintptr_t)mz); 616 if (ret == 0) /* should not happen */ 617 ret = -ENOBUFS; 618 if (ret < 0) { 619 rte_memzone_free(mz); 620 goto fail; 621 } 622 } 623 624 rte_mempool_trace_populate_default(mp); 625 return mp->size; 626 627 fail: 628 rte_mempool_free_memchunks(mp); 629 return ret; 630 } 631 632 /* return the memory size required for mempool objects in anonymous mem */ 633 static ssize_t 634 get_anon_size(const struct rte_mempool *mp) 635 { 636 ssize_t size; 637 size_t pg_sz, pg_shift; 638 size_t min_chunk_size; 639 size_t align; 640 641 pg_sz = rte_mem_page_size(); 642 pg_shift = rte_bsf32(pg_sz); 643 size = rte_mempool_ops_calc_mem_size(mp, mp->size, pg_shift, 644 &min_chunk_size, &align); 645 646 return size; 647 } 648 649 /* unmap a memory zone mapped by rte_mempool_populate_anon() */ 650 static void 651 rte_mempool_memchunk_anon_free(struct rte_mempool_memhdr *memhdr, 652 void *opaque) 653 { 654 ssize_t size; 655 656 /* 657 * Calculate size since memhdr->len has contiguous chunk length 658 * which may be smaller if anon map is split into many contiguous 659 * chunks. Result must be the same as we calculated on populate. 660 */ 661 size = get_anon_size(memhdr->mp); 662 if (size < 0) 663 return; 664 665 rte_mem_unmap(opaque, size); 666 } 667 668 /* populate the mempool with an anonymous mapping */ 669 int 670 rte_mempool_populate_anon(struct rte_mempool *mp) 671 { 672 ssize_t size; 673 int ret; 674 char *addr; 675 676 /* mempool is already populated, error */ 677 if ((!STAILQ_EMPTY(&mp->mem_list)) || mp->nb_mem_chunks != 0) { 678 rte_errno = EINVAL; 679 return 0; 680 } 681 682 ret = mempool_ops_alloc_once(mp); 683 if (ret < 0) { 684 rte_errno = -ret; 685 return 0; 686 } 687 688 size = get_anon_size(mp); 689 if (size < 0) { 690 rte_errno = -size; 691 return 0; 692 } 693 694 /* get chunk of virtually continuous memory */ 695 addr = rte_mem_map(NULL, size, RTE_PROT_READ | RTE_PROT_WRITE, 696 RTE_MAP_SHARED | RTE_MAP_ANONYMOUS, -1, 0); 697 if (addr == NULL) 698 return 0; 699 /* can't use MMAP_LOCKED, it does not exist on BSD */ 700 if (rte_mem_lock(addr, size) < 0) { 701 rte_mem_unmap(addr, size); 702 return 0; 703 } 704 705 ret = rte_mempool_populate_virt(mp, addr, size, rte_mem_page_size(), 706 rte_mempool_memchunk_anon_free, addr); 707 if (ret == 0) /* should not happen */ 708 ret = -ENOBUFS; 709 if (ret < 0) { 710 rte_errno = -ret; 711 goto fail; 712 } 713 714 rte_mempool_trace_populate_anon(mp); 715 return mp->populated_size; 716 717 fail: 718 rte_mempool_free_memchunks(mp); 719 return 0; 720 } 721 722 /* free a mempool */ 723 void 724 rte_mempool_free(struct rte_mempool *mp) 725 { 726 struct rte_mempool_list *mempool_list = NULL; 727 struct rte_tailq_entry *te; 728 729 if (mp == NULL) 730 return; 731 732 mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); 733 rte_mcfg_tailq_write_lock(); 734 /* find out tailq entry */ 735 TAILQ_FOREACH(te, mempool_list, next) { 736 if (te->data == (void *)mp) 737 break; 738 } 739 740 if (te != NULL) { 741 TAILQ_REMOVE(mempool_list, te, next); 742 rte_free(te); 743 } 744 rte_mcfg_tailq_write_unlock(); 745 746 mempool_event_callback_invoke(RTE_MEMPOOL_EVENT_DESTROY, mp); 747 rte_mempool_trace_free(mp); 748 rte_mempool_free_memchunks(mp); 749 rte_mempool_ops_free(mp); 750 rte_memzone_free(mp->mz); 751 } 752 753 static void 754 mempool_cache_init(struct rte_mempool_cache *cache, uint32_t size) 755 { 756 cache->size = size; 757 cache->flushthresh = CALC_CACHE_FLUSHTHRESH(size); 758 cache->len = 0; 759 } 760 761 /* 762 * Create and initialize a cache for objects that are retrieved from and 763 * returned to an underlying mempool. This structure is identical to the 764 * local_cache[lcore_id] pointed to by the mempool structure. 765 */ 766 struct rte_mempool_cache * 767 rte_mempool_cache_create(uint32_t size, int socket_id) 768 { 769 struct rte_mempool_cache *cache; 770 771 if (size == 0 || size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 772 rte_errno = EINVAL; 773 return NULL; 774 } 775 776 cache = rte_zmalloc_socket("MEMPOOL_CACHE", sizeof(*cache), 777 RTE_CACHE_LINE_SIZE, socket_id); 778 if (cache == NULL) { 779 RTE_LOG(ERR, MEMPOOL, "Cannot allocate mempool cache.\n"); 780 rte_errno = ENOMEM; 781 return NULL; 782 } 783 784 mempool_cache_init(cache, size); 785 786 rte_mempool_trace_cache_create(size, socket_id, cache); 787 return cache; 788 } 789 790 /* 791 * Free a cache. It's the responsibility of the user to make sure that any 792 * remaining objects in the cache are flushed to the corresponding 793 * mempool. 794 */ 795 void 796 rte_mempool_cache_free(struct rte_mempool_cache *cache) 797 { 798 rte_mempool_trace_cache_free(cache); 799 rte_free(cache); 800 } 801 802 /* create an empty mempool */ 803 struct rte_mempool * 804 rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, 805 unsigned cache_size, unsigned private_data_size, 806 int socket_id, unsigned flags) 807 { 808 char mz_name[RTE_MEMZONE_NAMESIZE]; 809 struct rte_mempool_list *mempool_list; 810 struct rte_mempool *mp = NULL; 811 struct rte_tailq_entry *te = NULL; 812 const struct rte_memzone *mz = NULL; 813 size_t mempool_size; 814 unsigned int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; 815 struct rte_mempool_objsz objsz; 816 unsigned lcore_id; 817 int ret; 818 819 /* compilation-time checks */ 820 RTE_BUILD_BUG_ON((sizeof(struct rte_mempool) & 821 RTE_CACHE_LINE_MASK) != 0); 822 RTE_BUILD_BUG_ON((sizeof(struct rte_mempool_cache) & 823 RTE_CACHE_LINE_MASK) != 0); 824 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 825 RTE_BUILD_BUG_ON((sizeof(struct rte_mempool_debug_stats) & 826 RTE_CACHE_LINE_MASK) != 0); 827 RTE_BUILD_BUG_ON((offsetof(struct rte_mempool, stats) & 828 RTE_CACHE_LINE_MASK) != 0); 829 #endif 830 831 mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); 832 833 /* asked for zero items */ 834 if (n == 0) { 835 rte_errno = EINVAL; 836 return NULL; 837 } 838 839 /* asked cache too big */ 840 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE || 841 CALC_CACHE_FLUSHTHRESH(cache_size) > n) { 842 rte_errno = EINVAL; 843 return NULL; 844 } 845 846 /* enforce only user flags are passed by the application */ 847 if ((flags & ~RTE_MEMPOOL_VALID_USER_FLAGS) != 0) { 848 rte_errno = EINVAL; 849 return NULL; 850 } 851 852 /* 853 * No objects in the pool can be used for IO until it's populated 854 * with at least some objects with valid IOVA. 855 */ 856 flags |= RTE_MEMPOOL_F_NON_IO; 857 858 /* "no cache align" imply "no spread" */ 859 if (flags & RTE_MEMPOOL_F_NO_CACHE_ALIGN) 860 flags |= RTE_MEMPOOL_F_NO_SPREAD; 861 862 /* calculate mempool object sizes. */ 863 if (!rte_mempool_calc_obj_size(elt_size, flags, &objsz)) { 864 rte_errno = EINVAL; 865 return NULL; 866 } 867 868 rte_mcfg_mempool_write_lock(); 869 870 /* 871 * reserve a memory zone for this mempool: private data is 872 * cache-aligned 873 */ 874 private_data_size = (private_data_size + 875 RTE_MEMPOOL_ALIGN_MASK) & (~RTE_MEMPOOL_ALIGN_MASK); 876 877 878 /* try to allocate tailq entry */ 879 te = rte_zmalloc("MEMPOOL_TAILQ_ENTRY", sizeof(*te), 0); 880 if (te == NULL) { 881 RTE_LOG(ERR, MEMPOOL, "Cannot allocate tailq entry!\n"); 882 goto exit_unlock; 883 } 884 885 mempool_size = RTE_MEMPOOL_HEADER_SIZE(mp, cache_size); 886 mempool_size += private_data_size; 887 mempool_size = RTE_ALIGN_CEIL(mempool_size, RTE_MEMPOOL_ALIGN); 888 889 ret = snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT, name); 890 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 891 rte_errno = ENAMETOOLONG; 892 goto exit_unlock; 893 } 894 895 mz = rte_memzone_reserve(mz_name, mempool_size, socket_id, mz_flags); 896 if (mz == NULL) 897 goto exit_unlock; 898 899 /* init the mempool structure */ 900 mp = mz->addr; 901 memset(mp, 0, RTE_MEMPOOL_HEADER_SIZE(mp, cache_size)); 902 ret = strlcpy(mp->name, name, sizeof(mp->name)); 903 if (ret < 0 || ret >= (int)sizeof(mp->name)) { 904 rte_errno = ENAMETOOLONG; 905 goto exit_unlock; 906 } 907 mp->mz = mz; 908 mp->size = n; 909 mp->flags = flags; 910 mp->socket_id = socket_id; 911 mp->elt_size = objsz.elt_size; 912 mp->header_size = objsz.header_size; 913 mp->trailer_size = objsz.trailer_size; 914 /* Size of default caches, zero means disabled. */ 915 mp->cache_size = cache_size; 916 mp->private_data_size = private_data_size; 917 STAILQ_INIT(&mp->elt_list); 918 STAILQ_INIT(&mp->mem_list); 919 920 /* 921 * local_cache pointer is set even if cache_size is zero. 922 * The local_cache points to just past the elt_pa[] array. 923 */ 924 mp->local_cache = (struct rte_mempool_cache *) 925 RTE_PTR_ADD(mp, RTE_MEMPOOL_HEADER_SIZE(mp, 0)); 926 927 /* Init all default caches. */ 928 if (cache_size != 0) { 929 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) 930 mempool_cache_init(&mp->local_cache[lcore_id], 931 cache_size); 932 } 933 934 te->data = mp; 935 936 rte_mcfg_tailq_write_lock(); 937 TAILQ_INSERT_TAIL(mempool_list, te, next); 938 rte_mcfg_tailq_write_unlock(); 939 rte_mcfg_mempool_write_unlock(); 940 941 rte_mempool_trace_create_empty(name, n, elt_size, cache_size, 942 private_data_size, flags, mp); 943 return mp; 944 945 exit_unlock: 946 rte_mcfg_mempool_write_unlock(); 947 rte_free(te); 948 rte_mempool_free(mp); 949 return NULL; 950 } 951 952 /* create the mempool */ 953 struct rte_mempool * 954 rte_mempool_create(const char *name, unsigned n, unsigned elt_size, 955 unsigned cache_size, unsigned private_data_size, 956 rte_mempool_ctor_t *mp_init, void *mp_init_arg, 957 rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, 958 int socket_id, unsigned flags) 959 { 960 int ret; 961 struct rte_mempool *mp; 962 963 mp = rte_mempool_create_empty(name, n, elt_size, cache_size, 964 private_data_size, socket_id, flags); 965 if (mp == NULL) 966 return NULL; 967 968 /* 969 * Since we have 4 combinations of the SP/SC/MP/MC examine the flags to 970 * set the correct index into the table of ops structs. 971 */ 972 if ((flags & RTE_MEMPOOL_F_SP_PUT) && (flags & RTE_MEMPOOL_F_SC_GET)) 973 ret = rte_mempool_set_ops_byname(mp, "ring_sp_sc", NULL); 974 else if (flags & RTE_MEMPOOL_F_SP_PUT) 975 ret = rte_mempool_set_ops_byname(mp, "ring_sp_mc", NULL); 976 else if (flags & RTE_MEMPOOL_F_SC_GET) 977 ret = rte_mempool_set_ops_byname(mp, "ring_mp_sc", NULL); 978 else 979 ret = rte_mempool_set_ops_byname(mp, "ring_mp_mc", NULL); 980 981 if (ret) 982 goto fail; 983 984 /* call the mempool priv initializer */ 985 if (mp_init) 986 mp_init(mp, mp_init_arg); 987 988 if (rte_mempool_populate_default(mp) < 0) 989 goto fail; 990 991 /* call the object initializers */ 992 if (obj_init) 993 rte_mempool_obj_iter(mp, obj_init, obj_init_arg); 994 995 rte_mempool_trace_create(name, n, elt_size, cache_size, 996 private_data_size, mp_init, mp_init_arg, obj_init, 997 obj_init_arg, flags, mp); 998 return mp; 999 1000 fail: 1001 rte_mempool_free(mp); 1002 return NULL; 1003 } 1004 1005 /* Return the number of entries in the mempool */ 1006 unsigned int 1007 rte_mempool_avail_count(const struct rte_mempool *mp) 1008 { 1009 unsigned count; 1010 unsigned lcore_id; 1011 1012 count = rte_mempool_ops_get_count(mp); 1013 1014 if (mp->cache_size == 0) 1015 return count; 1016 1017 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) 1018 count += mp->local_cache[lcore_id].len; 1019 1020 /* 1021 * due to race condition (access to len is not locked), the 1022 * total can be greater than size... so fix the result 1023 */ 1024 if (count > mp->size) 1025 return mp->size; 1026 return count; 1027 } 1028 1029 /* return the number of entries allocated from the mempool */ 1030 unsigned int 1031 rte_mempool_in_use_count(const struct rte_mempool *mp) 1032 { 1033 return mp->size - rte_mempool_avail_count(mp); 1034 } 1035 1036 /* dump the cache status */ 1037 static unsigned 1038 rte_mempool_dump_cache(FILE *f, const struct rte_mempool *mp) 1039 { 1040 unsigned lcore_id; 1041 unsigned count = 0; 1042 unsigned cache_count; 1043 1044 fprintf(f, " internal cache infos:\n"); 1045 fprintf(f, " cache_size=%"PRIu32"\n", mp->cache_size); 1046 1047 if (mp->cache_size == 0) 1048 return count; 1049 1050 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1051 cache_count = mp->local_cache[lcore_id].len; 1052 fprintf(f, " cache_count[%u]=%"PRIu32"\n", 1053 lcore_id, cache_count); 1054 count += cache_count; 1055 } 1056 fprintf(f, " total_cache_count=%u\n", count); 1057 return count; 1058 } 1059 1060 #ifndef __INTEL_COMPILER 1061 #pragma GCC diagnostic ignored "-Wcast-qual" 1062 #endif 1063 1064 /* check and update cookies or panic (internal) */ 1065 void rte_mempool_check_cookies(const struct rte_mempool *mp, 1066 void * const *obj_table_const, unsigned n, int free) 1067 { 1068 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 1069 struct rte_mempool_objhdr *hdr; 1070 struct rte_mempool_objtlr *tlr; 1071 uint64_t cookie; 1072 void *tmp; 1073 void *obj; 1074 void **obj_table; 1075 1076 /* Force to drop the "const" attribute. This is done only when 1077 * DEBUG is enabled */ 1078 tmp = (void *) obj_table_const; 1079 obj_table = tmp; 1080 1081 while (n--) { 1082 obj = obj_table[n]; 1083 1084 if (rte_mempool_from_obj(obj) != mp) 1085 rte_panic("MEMPOOL: object is owned by another " 1086 "mempool\n"); 1087 1088 hdr = rte_mempool_get_header(obj); 1089 cookie = hdr->cookie; 1090 1091 if (free == 0) { 1092 if (cookie != RTE_MEMPOOL_HEADER_COOKIE1) { 1093 RTE_LOG(CRIT, MEMPOOL, 1094 "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", 1095 obj, (const void *) mp, cookie); 1096 rte_panic("MEMPOOL: bad header cookie (put)\n"); 1097 } 1098 hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE2; 1099 } else if (free == 1) { 1100 if (cookie != RTE_MEMPOOL_HEADER_COOKIE2) { 1101 RTE_LOG(CRIT, MEMPOOL, 1102 "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", 1103 obj, (const void *) mp, cookie); 1104 rte_panic("MEMPOOL: bad header cookie (get)\n"); 1105 } 1106 hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE1; 1107 } else if (free == 2) { 1108 if (cookie != RTE_MEMPOOL_HEADER_COOKIE1 && 1109 cookie != RTE_MEMPOOL_HEADER_COOKIE2) { 1110 RTE_LOG(CRIT, MEMPOOL, 1111 "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", 1112 obj, (const void *) mp, cookie); 1113 rte_panic("MEMPOOL: bad header cookie (audit)\n"); 1114 } 1115 } 1116 tlr = rte_mempool_get_trailer(obj); 1117 cookie = tlr->cookie; 1118 if (cookie != RTE_MEMPOOL_TRAILER_COOKIE) { 1119 RTE_LOG(CRIT, MEMPOOL, 1120 "obj=%p, mempool=%p, cookie=%" PRIx64 "\n", 1121 obj, (const void *) mp, cookie); 1122 rte_panic("MEMPOOL: bad trailer cookie\n"); 1123 } 1124 } 1125 #else 1126 RTE_SET_USED(mp); 1127 RTE_SET_USED(obj_table_const); 1128 RTE_SET_USED(n); 1129 RTE_SET_USED(free); 1130 #endif 1131 } 1132 1133 void 1134 rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp, 1135 void * const *first_obj_table_const, unsigned int n, int free) 1136 { 1137 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 1138 struct rte_mempool_info info; 1139 const size_t total_elt_sz = 1140 mp->header_size + mp->elt_size + mp->trailer_size; 1141 unsigned int i, j; 1142 1143 rte_mempool_ops_get_info(mp, &info); 1144 1145 for (i = 0; i < n; ++i) { 1146 void *first_obj = first_obj_table_const[i]; 1147 1148 for (j = 0; j < info.contig_block_size; ++j) { 1149 void *obj; 1150 1151 obj = (void *)((uintptr_t)first_obj + j * total_elt_sz); 1152 rte_mempool_check_cookies(mp, &obj, 1, free); 1153 } 1154 } 1155 #else 1156 RTE_SET_USED(mp); 1157 RTE_SET_USED(first_obj_table_const); 1158 RTE_SET_USED(n); 1159 RTE_SET_USED(free); 1160 #endif 1161 } 1162 1163 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 1164 static void 1165 mempool_obj_audit(struct rte_mempool *mp, __rte_unused void *opaque, 1166 void *obj, __rte_unused unsigned idx) 1167 { 1168 RTE_MEMPOOL_CHECK_COOKIES(mp, &obj, 1, 2); 1169 } 1170 1171 static void 1172 mempool_audit_cookies(struct rte_mempool *mp) 1173 { 1174 unsigned num; 1175 1176 num = rte_mempool_obj_iter(mp, mempool_obj_audit, NULL); 1177 if (num != mp->size) { 1178 rte_panic("rte_mempool_obj_iter(mempool=%p, size=%u) " 1179 "iterated only over %u elements\n", 1180 mp, mp->size, num); 1181 } 1182 } 1183 #else 1184 #define mempool_audit_cookies(mp) do {} while(0) 1185 #endif 1186 1187 #ifndef __INTEL_COMPILER 1188 #pragma GCC diagnostic error "-Wcast-qual" 1189 #endif 1190 1191 /* check cookies before and after objects */ 1192 static void 1193 mempool_audit_cache(const struct rte_mempool *mp) 1194 { 1195 /* check cache size consistency */ 1196 unsigned lcore_id; 1197 1198 if (mp->cache_size == 0) 1199 return; 1200 1201 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1202 const struct rte_mempool_cache *cache; 1203 cache = &mp->local_cache[lcore_id]; 1204 if (cache->len > RTE_DIM(cache->objs)) { 1205 RTE_LOG(CRIT, MEMPOOL, "badness on cache[%u]\n", 1206 lcore_id); 1207 rte_panic("MEMPOOL: invalid cache len\n"); 1208 } 1209 } 1210 } 1211 1212 /* check the consistency of mempool (size, cookies, ...) */ 1213 void 1214 rte_mempool_audit(struct rte_mempool *mp) 1215 { 1216 mempool_audit_cache(mp); 1217 mempool_audit_cookies(mp); 1218 1219 /* For case where mempool DEBUG is not set, and cache size is 0 */ 1220 RTE_SET_USED(mp); 1221 } 1222 1223 /* dump the status of the mempool on the console */ 1224 void 1225 rte_mempool_dump(FILE *f, struct rte_mempool *mp) 1226 { 1227 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 1228 struct rte_mempool_info info; 1229 struct rte_mempool_debug_stats sum; 1230 unsigned lcore_id; 1231 #endif 1232 struct rte_mempool_memhdr *memhdr; 1233 struct rte_mempool_ops *ops; 1234 unsigned common_count; 1235 unsigned cache_count; 1236 size_t mem_len = 0; 1237 1238 RTE_ASSERT(f != NULL); 1239 RTE_ASSERT(mp != NULL); 1240 1241 fprintf(f, "mempool <%s>@%p\n", mp->name, mp); 1242 fprintf(f, " flags=%x\n", mp->flags); 1243 fprintf(f, " socket_id=%d\n", mp->socket_id); 1244 fprintf(f, " pool=%p\n", mp->pool_data); 1245 fprintf(f, " iova=0x%" PRIx64 "\n", mp->mz->iova); 1246 fprintf(f, " nb_mem_chunks=%u\n", mp->nb_mem_chunks); 1247 fprintf(f, " size=%"PRIu32"\n", mp->size); 1248 fprintf(f, " populated_size=%"PRIu32"\n", mp->populated_size); 1249 fprintf(f, " header_size=%"PRIu32"\n", mp->header_size); 1250 fprintf(f, " elt_size=%"PRIu32"\n", mp->elt_size); 1251 fprintf(f, " trailer_size=%"PRIu32"\n", mp->trailer_size); 1252 fprintf(f, " total_obj_size=%"PRIu32"\n", 1253 mp->header_size + mp->elt_size + mp->trailer_size); 1254 1255 fprintf(f, " private_data_size=%"PRIu32"\n", mp->private_data_size); 1256 1257 fprintf(f, " ops_index=%d\n", mp->ops_index); 1258 ops = rte_mempool_get_ops(mp->ops_index); 1259 fprintf(f, " ops_name: <%s>\n", (ops != NULL) ? ops->name : "NA"); 1260 1261 STAILQ_FOREACH(memhdr, &mp->mem_list, next) 1262 mem_len += memhdr->len; 1263 if (mem_len != 0) { 1264 fprintf(f, " avg bytes/object=%#Lf\n", 1265 (long double)mem_len / mp->size); 1266 } 1267 1268 cache_count = rte_mempool_dump_cache(f, mp); 1269 common_count = rte_mempool_ops_get_count(mp); 1270 if ((cache_count + common_count) > mp->size) 1271 common_count = mp->size - cache_count; 1272 fprintf(f, " common_pool_count=%u\n", common_count); 1273 1274 /* sum and dump statistics */ 1275 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG 1276 rte_mempool_ops_get_info(mp, &info); 1277 memset(&sum, 0, sizeof(sum)); 1278 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { 1279 sum.put_bulk += mp->stats[lcore_id].put_bulk; 1280 sum.put_objs += mp->stats[lcore_id].put_objs; 1281 sum.put_common_pool_bulk += mp->stats[lcore_id].put_common_pool_bulk; 1282 sum.put_common_pool_objs += mp->stats[lcore_id].put_common_pool_objs; 1283 sum.get_common_pool_bulk += mp->stats[lcore_id].get_common_pool_bulk; 1284 sum.get_common_pool_objs += mp->stats[lcore_id].get_common_pool_objs; 1285 sum.get_success_bulk += mp->stats[lcore_id].get_success_bulk; 1286 sum.get_success_objs += mp->stats[lcore_id].get_success_objs; 1287 sum.get_fail_bulk += mp->stats[lcore_id].get_fail_bulk; 1288 sum.get_fail_objs += mp->stats[lcore_id].get_fail_objs; 1289 sum.get_success_blks += mp->stats[lcore_id].get_success_blks; 1290 sum.get_fail_blks += mp->stats[lcore_id].get_fail_blks; 1291 } 1292 fprintf(f, " stats:\n"); 1293 fprintf(f, " put_bulk=%"PRIu64"\n", sum.put_bulk); 1294 fprintf(f, " put_objs=%"PRIu64"\n", sum.put_objs); 1295 fprintf(f, " put_common_pool_bulk=%"PRIu64"\n", sum.put_common_pool_bulk); 1296 fprintf(f, " put_common_pool_objs=%"PRIu64"\n", sum.put_common_pool_objs); 1297 fprintf(f, " get_common_pool_bulk=%"PRIu64"\n", sum.get_common_pool_bulk); 1298 fprintf(f, " get_common_pool_objs=%"PRIu64"\n", sum.get_common_pool_objs); 1299 fprintf(f, " get_success_bulk=%"PRIu64"\n", sum.get_success_bulk); 1300 fprintf(f, " get_success_objs=%"PRIu64"\n", sum.get_success_objs); 1301 fprintf(f, " get_fail_bulk=%"PRIu64"\n", sum.get_fail_bulk); 1302 fprintf(f, " get_fail_objs=%"PRIu64"\n", sum.get_fail_objs); 1303 if (info.contig_block_size > 0) { 1304 fprintf(f, " get_success_blks=%"PRIu64"\n", 1305 sum.get_success_blks); 1306 fprintf(f, " get_fail_blks=%"PRIu64"\n", sum.get_fail_blks); 1307 } 1308 #else 1309 fprintf(f, " no statistics available\n"); 1310 #endif 1311 1312 rte_mempool_audit(mp); 1313 } 1314 1315 /* dump the status of all mempools on the console */ 1316 void 1317 rte_mempool_list_dump(FILE *f) 1318 { 1319 struct rte_mempool *mp = NULL; 1320 struct rte_tailq_entry *te; 1321 struct rte_mempool_list *mempool_list; 1322 1323 mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); 1324 1325 rte_mcfg_mempool_read_lock(); 1326 1327 TAILQ_FOREACH(te, mempool_list, next) { 1328 mp = (struct rte_mempool *) te->data; 1329 rte_mempool_dump(f, mp); 1330 } 1331 1332 rte_mcfg_mempool_read_unlock(); 1333 } 1334 1335 /* search a mempool from its name */ 1336 struct rte_mempool * 1337 rte_mempool_lookup(const char *name) 1338 { 1339 struct rte_mempool *mp = NULL; 1340 struct rte_tailq_entry *te; 1341 struct rte_mempool_list *mempool_list; 1342 1343 mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); 1344 1345 rte_mcfg_mempool_read_lock(); 1346 1347 TAILQ_FOREACH(te, mempool_list, next) { 1348 mp = (struct rte_mempool *) te->data; 1349 if (strncmp(name, mp->name, RTE_MEMPOOL_NAMESIZE) == 0) 1350 break; 1351 } 1352 1353 rte_mcfg_mempool_read_unlock(); 1354 1355 if (te == NULL) { 1356 rte_errno = ENOENT; 1357 return NULL; 1358 } 1359 1360 return mp; 1361 } 1362 1363 void rte_mempool_walk(void (*func)(struct rte_mempool *, void *), 1364 void *arg) 1365 { 1366 struct rte_tailq_entry *te = NULL; 1367 struct rte_mempool_list *mempool_list; 1368 void *tmp_te; 1369 1370 mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list); 1371 1372 rte_mcfg_mempool_read_lock(); 1373 1374 RTE_TAILQ_FOREACH_SAFE(te, mempool_list, next, tmp_te) { 1375 (*func)((struct rte_mempool *) te->data, arg); 1376 } 1377 1378 rte_mcfg_mempool_read_unlock(); 1379 } 1380 1381 struct mempool_callback_data { 1382 rte_mempool_event_callback *func; 1383 void *user_data; 1384 }; 1385 1386 static void 1387 mempool_event_callback_invoke(enum rte_mempool_event event, 1388 struct rte_mempool *mp) 1389 { 1390 struct mempool_callback_list *list; 1391 struct rte_tailq_entry *te; 1392 void *tmp_te; 1393 1394 rte_mcfg_tailq_read_lock(); 1395 list = RTE_TAILQ_CAST(callback_tailq.head, mempool_callback_list); 1396 RTE_TAILQ_FOREACH_SAFE(te, list, next, tmp_te) { 1397 struct mempool_callback_data *cb = te->data; 1398 rte_mcfg_tailq_read_unlock(); 1399 cb->func(event, mp, cb->user_data); 1400 rte_mcfg_tailq_read_lock(); 1401 } 1402 rte_mcfg_tailq_read_unlock(); 1403 } 1404 1405 int 1406 rte_mempool_event_callback_register(rte_mempool_event_callback *func, 1407 void *user_data) 1408 { 1409 struct mempool_callback_list *list; 1410 struct rte_tailq_entry *te = NULL; 1411 struct mempool_callback_data *cb; 1412 void *tmp_te; 1413 int ret; 1414 1415 if (func == NULL) { 1416 rte_errno = EINVAL; 1417 return -rte_errno; 1418 } 1419 1420 rte_mcfg_tailq_write_lock(); 1421 list = RTE_TAILQ_CAST(callback_tailq.head, mempool_callback_list); 1422 RTE_TAILQ_FOREACH_SAFE(te, list, next, tmp_te) { 1423 cb = te->data; 1424 if (cb->func == func && cb->user_data == user_data) { 1425 ret = -EEXIST; 1426 goto exit; 1427 } 1428 } 1429 1430 te = rte_zmalloc("mempool_cb_tail_entry", sizeof(*te), 0); 1431 if (te == NULL) { 1432 RTE_LOG(ERR, MEMPOOL, 1433 "Cannot allocate event callback tailq entry!\n"); 1434 ret = -ENOMEM; 1435 goto exit; 1436 } 1437 1438 cb = rte_malloc("mempool_cb_data", sizeof(*cb), 0); 1439 if (cb == NULL) { 1440 RTE_LOG(ERR, MEMPOOL, 1441 "Cannot allocate event callback!\n"); 1442 rte_free(te); 1443 ret = -ENOMEM; 1444 goto exit; 1445 } 1446 1447 cb->func = func; 1448 cb->user_data = user_data; 1449 te->data = cb; 1450 TAILQ_INSERT_TAIL(list, te, next); 1451 ret = 0; 1452 1453 exit: 1454 rte_mcfg_tailq_write_unlock(); 1455 rte_errno = -ret; 1456 return ret; 1457 } 1458 1459 int 1460 rte_mempool_event_callback_unregister(rte_mempool_event_callback *func, 1461 void *user_data) 1462 { 1463 struct mempool_callback_list *list; 1464 struct rte_tailq_entry *te = NULL; 1465 struct mempool_callback_data *cb; 1466 int ret = -ENOENT; 1467 1468 rte_mcfg_tailq_write_lock(); 1469 list = RTE_TAILQ_CAST(callback_tailq.head, mempool_callback_list); 1470 TAILQ_FOREACH(te, list, next) { 1471 cb = te->data; 1472 if (cb->func == func && cb->user_data == user_data) { 1473 TAILQ_REMOVE(list, te, next); 1474 ret = 0; 1475 break; 1476 } 1477 } 1478 rte_mcfg_tailq_write_unlock(); 1479 1480 if (ret == 0) { 1481 rte_free(te); 1482 rte_free(cb); 1483 } 1484 rte_errno = -ret; 1485 return ret; 1486 } 1487 1488 static void 1489 mempool_list_cb(struct rte_mempool *mp, void *arg) 1490 { 1491 struct rte_tel_data *d = (struct rte_tel_data *)arg; 1492 1493 rte_tel_data_add_array_string(d, mp->name); 1494 } 1495 1496 static int 1497 mempool_handle_list(const char *cmd __rte_unused, 1498 const char *params __rte_unused, struct rte_tel_data *d) 1499 { 1500 rte_tel_data_start_array(d, RTE_TEL_STRING_VAL); 1501 rte_mempool_walk(mempool_list_cb, d); 1502 return 0; 1503 } 1504 1505 struct mempool_info_cb_arg { 1506 char *pool_name; 1507 struct rte_tel_data *d; 1508 }; 1509 1510 static void 1511 mempool_info_cb(struct rte_mempool *mp, void *arg) 1512 { 1513 struct mempool_info_cb_arg *info = (struct mempool_info_cb_arg *)arg; 1514 const struct rte_memzone *mz; 1515 1516 if (strncmp(mp->name, info->pool_name, RTE_MEMZONE_NAMESIZE)) 1517 return; 1518 1519 rte_tel_data_add_dict_string(info->d, "name", mp->name); 1520 rte_tel_data_add_dict_int(info->d, "pool_id", mp->pool_id); 1521 rte_tel_data_add_dict_int(info->d, "flags", mp->flags); 1522 rte_tel_data_add_dict_int(info->d, "socket_id", mp->socket_id); 1523 rte_tel_data_add_dict_int(info->d, "size", mp->size); 1524 rte_tel_data_add_dict_int(info->d, "cache_size", mp->cache_size); 1525 rte_tel_data_add_dict_int(info->d, "elt_size", mp->elt_size); 1526 rte_tel_data_add_dict_int(info->d, "header_size", mp->header_size); 1527 rte_tel_data_add_dict_int(info->d, "trailer_size", mp->trailer_size); 1528 rte_tel_data_add_dict_int(info->d, "private_data_size", 1529 mp->private_data_size); 1530 rte_tel_data_add_dict_int(info->d, "ops_index", mp->ops_index); 1531 rte_tel_data_add_dict_int(info->d, "populated_size", 1532 mp->populated_size); 1533 1534 mz = mp->mz; 1535 rte_tel_data_add_dict_string(info->d, "mz_name", mz->name); 1536 rte_tel_data_add_dict_int(info->d, "mz_len", mz->len); 1537 rte_tel_data_add_dict_int(info->d, "mz_hugepage_sz", 1538 mz->hugepage_sz); 1539 rte_tel_data_add_dict_int(info->d, "mz_socket_id", mz->socket_id); 1540 rte_tel_data_add_dict_int(info->d, "mz_flags", mz->flags); 1541 } 1542 1543 static int 1544 mempool_handle_info(const char *cmd __rte_unused, const char *params, 1545 struct rte_tel_data *d) 1546 { 1547 struct mempool_info_cb_arg mp_arg; 1548 char name[RTE_MEMZONE_NAMESIZE]; 1549 1550 if (!params || strlen(params) == 0) 1551 return -EINVAL; 1552 1553 rte_strlcpy(name, params, RTE_MEMZONE_NAMESIZE); 1554 1555 rte_tel_data_start_dict(d); 1556 mp_arg.pool_name = name; 1557 mp_arg.d = d; 1558 rte_mempool_walk(mempool_info_cb, &mp_arg); 1559 1560 return 0; 1561 } 1562 1563 RTE_INIT(mempool_init_telemetry) 1564 { 1565 rte_telemetry_register_cmd("/mempool/list", mempool_handle_list, 1566 "Returns list of available mempool. Takes no parameters"); 1567 rte_telemetry_register_cmd("/mempool/info", mempool_handle_info, 1568 "Returns mempool info. Parameters: pool_name"); 1569 } 1570