140 	struct rte_mempool *mp;          /**< The mempool owning the object. */  member
185 	struct rte_mempool *mp;  /**< The mempool owning the chunk */  member
273 #define __MEMPOOL_STAT_ADD(mp, name, n) do {                    \  argument
276 			mp->stats[__lcore_id].name##_objs += n;	\
277 			mp->stats[__lcore_id].name##_bulk += 1;	\
280 #define __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, name, n) do {                    \  argument
283 			mp->stats[__lcore_id].name##_blks += n;	\
284 			mp->stats[__lcore_id].name##_bulk += 1;	\
288 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)  argument
289 #define __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, name, n) do {} while (0)  argument
300 #define MEMPOOL_HEADER_SIZE(mp, cs) \  argument
301 	(sizeof(*(mp)) + (((cs) == 0) ? 0 : \
323 	return hdr->mp;  in rte_mempool_from_obj()
329 	struct rte_mempool *mp = rte_mempool_from_obj(obj);  in __mempool_get_trailer()  local
330 	return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);  in __mempool_get_trailer()
347 void rte_mempool_check_cookies(const struct rte_mempool *mp,
351 #define __mempool_check_cookies(mp, obj_table_const, n, free) \  argument
352 	rte_mempool_check_cookies(mp, obj_table_const, n, free)
354 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)  argument
372 void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
376 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \  argument
378 	rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
381 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \  argument
398 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
403 typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
408 typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
414 typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
420 typedef int (*rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp,
426 typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
451 typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
490 ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
501 ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
517 typedef void (rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp,
548 typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
590 int rte_mempool_op_populate_helper(struct rte_mempool *mp,
601 int rte_mempool_op_populate_default(struct rte_mempool *mp,
609 typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
691 rte_mempool_ops_alloc(struct rte_mempool *mp);
707 rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,  in rte_mempool_ops_dequeue_bulk()  argument
712 	rte_mempool_trace_ops_dequeue_bulk(mp, obj_table, n);  in rte_mempool_ops_dequeue_bulk()
713 	ops = rte_mempool_get_ops(mp->ops_index);  in rte_mempool_ops_dequeue_bulk()
714 	return ops->dequeue(mp, obj_table, n);  in rte_mempool_ops_dequeue_bulk()
731 rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,  in rte_mempool_ops_dequeue_contig_blocks()  argument
736 	ops = rte_mempool_get_ops(mp->ops_index);  in rte_mempool_ops_dequeue_contig_blocks()
738 	rte_mempool_trace_ops_dequeue_contig_blocks(mp, first_obj_table, n);  in rte_mempool_ops_dequeue_contig_blocks()
739 	return ops->dequeue_contig_blocks(mp, first_obj_table, n);  in rte_mempool_ops_dequeue_contig_blocks()
756 rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,  in rte_mempool_ops_enqueue_bulk()  argument
761 	rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);  in rte_mempool_ops_enqueue_bulk()
762 	ops = rte_mempool_get_ops(mp->ops_index);  in rte_mempool_ops_enqueue_bulk()
763 	return ops->enqueue(mp, obj_table, n);  in rte_mempool_ops_enqueue_bulk()
775 rte_mempool_ops_get_count(const struct rte_mempool *mp);
796 ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
823 int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
840 int rte_mempool_ops_get_info(const struct rte_mempool *mp,
850 rte_mempool_ops_free(struct rte_mempool *mp);
870 rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
901 typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
910 typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
1057 rte_mempool_free(struct rte_mempool *mp);
1089 int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1120 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
1137 int rte_mempool_populate_default(struct rte_mempool *mp);
1152 int rte_mempool_populate_anon(struct rte_mempool *mp);
1169 uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1187 uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1198 void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1238 rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)  in rte_mempool_default_cache()  argument
1240 	if (mp->cache_size == 0)  in rte_mempool_default_cache()
1246 	rte_mempool_trace_default_cache(mp, lcore_id,  in rte_mempool_default_cache()
1247 		&mp->local_cache[lcore_id]);  in rte_mempool_default_cache()
1248 	return &mp->local_cache[lcore_id];  in rte_mempool_default_cache()
1261 			struct rte_mempool *mp)  in rte_mempool_cache_flush()  argument
1264 		cache = rte_mempool_default_cache(mp, rte_lcore_id());  in rte_mempool_cache_flush()
1267 	rte_mempool_trace_cache_flush(cache, mp);  in rte_mempool_cache_flush()
1268 	rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);  in rte_mempool_cache_flush()
1285 __mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,  in __mempool_generic_put()  argument
1291 	__MEMPOOL_STAT_ADD(mp, put, n);  in __mempool_generic_put()
1312 		rte_mempool_ops_enqueue_bulk(mp, &cache->objs[cache->size],  in __mempool_generic_put()
1323 	if (rte_mempool_ops_enqueue_bulk(mp, obj_table, n) < 0)  in __mempool_generic_put()
1326 	rte_mempool_ops_enqueue_bulk(mp, obj_table, n);  in __mempool_generic_put()
1344 rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,  in rte_mempool_generic_put()  argument
1347 	rte_mempool_trace_generic_put(mp, obj_table, n, cache);  in rte_mempool_generic_put()
1348 	__mempool_check_cookies(mp, obj_table, n, 0);  in rte_mempool_generic_put()
1349 	__mempool_generic_put(mp, obj_table, n, cache);  in rte_mempool_generic_put()
1367 rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,  in rte_mempool_put_bulk()  argument
1371 	cache = rte_mempool_default_cache(mp, rte_lcore_id());  in rte_mempool_put_bulk()
1372 	rte_mempool_trace_put_bulk(mp, obj_table, n, cache);  in rte_mempool_put_bulk()
1373 	rte_mempool_generic_put(mp, obj_table, n, cache);  in rte_mempool_put_bulk()
1389 rte_mempool_put(struct rte_mempool *mp, void *obj)  in rte_mempool_put()  argument
1391 	rte_mempool_put_bulk(mp, &obj, 1);  in rte_mempool_put()
1409 __mempool_generic_get(struct rte_mempool *mp, void **obj_table,  in __mempool_generic_get()  argument
1428 		ret = rte_mempool_ops_dequeue_bulk(mp,  in __mempool_generic_get()
1449 	__MEMPOOL_STAT_ADD(mp, get_success, n);  in __mempool_generic_get()
1456 	ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, n);  in __mempool_generic_get()
1459 		__MEMPOOL_STAT_ADD(mp, get_fail, n);  in __mempool_generic_get()
1461 		__MEMPOOL_STAT_ADD(mp, get_success, n);  in __mempool_generic_get()
1487 rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,  in rte_mempool_generic_get()  argument
1491 	ret = __mempool_generic_get(mp, obj_table, n, cache);  in rte_mempool_generic_get()
1493 		__mempool_check_cookies(mp, obj_table, n, 1);  in rte_mempool_generic_get()
1494 	rte_mempool_trace_generic_get(mp, obj_table, n, cache);  in rte_mempool_generic_get()
1521 rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)  in rte_mempool_get_bulk()  argument
1524 	cache = rte_mempool_default_cache(mp, rte_lcore_id());  in rte_mempool_get_bulk()
1525 	rte_mempool_trace_get_bulk(mp, obj_table, n, cache);  in rte_mempool_get_bulk()
1526 	return rte_mempool_generic_get(mp, obj_table, n, cache);  in rte_mempool_get_bulk()
1550 rte_mempool_get(struct rte_mempool *mp, void **obj_p)  in rte_mempool_get()  argument
1552 	return rte_mempool_get_bulk(mp, obj_p, 1);  in rte_mempool_get()
1577 rte_mempool_get_contig_blocks(struct rte_mempool *mp,  in rte_mempool_get_contig_blocks()  argument
1582 	ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);  in rte_mempool_get_contig_blocks()
1584 		__MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, get_success, n);  in rte_mempool_get_contig_blocks()
1585 		__mempool_contig_blocks_check_cookies(mp, first_obj_table, n,  in rte_mempool_get_contig_blocks()
1588 		__MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, get_fail, n);  in rte_mempool_get_contig_blocks()
1591 	rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);  in rte_mempool_get_contig_blocks()
1607 unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1622 rte_mempool_in_use_count(const struct rte_mempool *mp);
1638 rte_mempool_full(const struct rte_mempool *mp)  in rte_mempool_full()  argument
1640 	return rte_mempool_avail_count(mp) == mp->size;  in rte_mempool_full()
1657 rte_mempool_empty(const struct rte_mempool *mp)  in rte_mempool_empty()  argument
1659 	return rte_mempool_avail_count(mp) == 0;  in rte_mempool_empty()
1691 void rte_mempool_audit(struct rte_mempool *mp);
1701 static inline void *rte_mempool_get_priv(struct rte_mempool *mp)  in rte_mempool_get_priv()  argument
1703 	return (char *)mp +  in rte_mempool_get_priv()
1704 		MEMPOOL_HEADER_SIZE(mp, mp->cache_size);  in rte_mempool_get_priv()
1765 rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);