155 mempool_add_elem(struct rte_mempool *mp, __rte_unused void *opaque,  in mempool_add_elem()  argument
163 	hdr->mp = mp;  in mempool_add_elem()
165 	STAILQ_INSERT_TAIL(&mp->elt_list, hdr, next);  in mempool_add_elem()
166 	mp->populated_size++;  in mempool_add_elem()
177 rte_mempool_obj_iter(struct rte_mempool *mp,  in rte_mempool_obj_iter()  argument
184 	STAILQ_FOREACH(hdr, &mp->elt_list, next) {  in rte_mempool_obj_iter()
186 		obj_cb(mp, obj_cb_arg, obj, n);  in rte_mempool_obj_iter()
195 rte_mempool_mem_iter(struct rte_mempool *mp,  in rte_mempool_mem_iter()  argument
201 	STAILQ_FOREACH(hdr, &mp->mem_list, next) {  in rte_mempool_mem_iter()
202 		mem_cb(mp, mem_cb_arg, hdr, n);  in rte_mempool_mem_iter()
269 rte_mempool_free_memchunks(struct rte_mempool *mp)  in rte_mempool_free_memchunks()  argument
274 	while (!STAILQ_EMPTY(&mp->elt_list)) {  in rte_mempool_free_memchunks()
275 		rte_mempool_ops_dequeue_bulk(mp, &elt, 1);  in rte_mempool_free_memchunks()
277 		STAILQ_REMOVE_HEAD(&mp->elt_list, next);  in rte_mempool_free_memchunks()
278 		mp->populated_size--;  in rte_mempool_free_memchunks()
281 	while (!STAILQ_EMPTY(&mp->mem_list)) {  in rte_mempool_free_memchunks()
282 		memhdr = STAILQ_FIRST(&mp->mem_list);  in rte_mempool_free_memchunks()
283 		STAILQ_REMOVE_HEAD(&mp->mem_list, next);  in rte_mempool_free_memchunks()
287 		mp->nb_mem_chunks--;  in rte_mempool_free_memchunks()
292 mempool_ops_alloc_once(struct rte_mempool *mp)  in mempool_ops_alloc_once()  argument
297 	if ((mp->flags & MEMPOOL_F_POOL_CREATED) == 0) {  in mempool_ops_alloc_once()
298 		ret = rte_mempool_ops_alloc(mp);  in mempool_ops_alloc_once()
301 		mp->flags |= MEMPOOL_F_POOL_CREATED;  in mempool_ops_alloc_once()
311 rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,  in rte_mempool_populate_iova()  argument
320 	ret = mempool_ops_alloc_once(mp);  in rte_mempool_populate_iova()
325 	if (mp->populated_size >= mp->size)  in rte_mempool_populate_iova()
332 	memhdr->mp = mp;  in rte_mempool_populate_iova()
339 	if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN)  in rte_mempool_populate_iova()
349 	i = rte_mempool_ops_populate(mp, mp->size - mp->populated_size,  in rte_mempool_populate_iova()
360 	STAILQ_INSERT_TAIL(&mp->mem_list, memhdr, next);  in rte_mempool_populate_iova()
361 	mp->nb_mem_chunks++;  in rte_mempool_populate_iova()
363 	rte_mempool_trace_populate_iova(mp, vaddr, iova, len, free_cb, opaque);  in rte_mempool_populate_iova()
388 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,  in rte_mempool_populate_virt()  argument
396 	if (mp->flags & MEMPOOL_F_NO_IOVA_CONTIG)  in rte_mempool_populate_virt()
397 		return rte_mempool_populate_iova(mp, addr, RTE_BAD_IOVA,  in rte_mempool_populate_virt()
401 		     mp->populated_size < mp->size; off += phys_len) {  in rte_mempool_populate_virt()
421 		ret = rte_mempool_populate_iova(mp, addr + off, iova,  in rte_mempool_populate_virt()
432 	rte_mempool_trace_populate_virt(mp, addr, len, pg_sz, free_cb, opaque);  in rte_mempool_populate_virt()
436 	rte_mempool_free_memchunks(mp);  in rte_mempool_populate_virt()
442 rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz)  in rte_mempool_get_page_size()  argument
449 	ret = rte_malloc_heap_socket_is_external(mp->socket_id);  in rte_mempool_get_page_size()
453 	need_iova_contig_obj = !(mp->flags & MEMPOOL_F_NO_IOVA_CONTIG);  in rte_mempool_get_page_size()
458 		*pg_sz = get_min_page_size(mp->socket_id);  in rte_mempool_get_page_size()
462 	rte_mempool_trace_get_page_size(mp, *pg_sz);  in rte_mempool_get_page_size()
471 rte_mempool_populate_default(struct rte_mempool *mp)  in rte_mempool_populate_default()  argument
484 	ret = mempool_ops_alloc_once(mp);  in rte_mempool_populate_default()
489 	if (mp->nb_mem_chunks != 0)  in rte_mempool_populate_default()
530 	need_iova_contig_obj = !(mp->flags & MEMPOOL_F_NO_IOVA_CONTIG);  in rte_mempool_populate_default()
531 	ret = rte_mempool_get_page_size(mp, &pg_sz);  in rte_mempool_populate_default()
538 	for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) {  in rte_mempool_populate_default()
542 			mp, n, pg_shift, &min_chunk_size, &align);  in rte_mempool_populate_default()
550 			RTE_MEMPOOL_MZ_FORMAT "_%d", mp->name, mz_id);  in rte_mempool_populate_default()
566 				mp->socket_id, mz_flags, align);  in rte_mempool_populate_default()
586 			ret = rte_mempool_populate_iova(mp, mz->addr,  in rte_mempool_populate_default()
591 			ret = rte_mempool_populate_virt(mp, mz->addr,  in rte_mempool_populate_default()
603 	rte_mempool_trace_populate_default(mp);  in rte_mempool_populate_default()
604 	return mp->size;  in rte_mempool_populate_default()
607 	rte_mempool_free_memchunks(mp);  in rte_mempool_populate_default()
613 get_anon_size(const struct rte_mempool *mp)  in get_anon_size()  argument
622 	size = rte_mempool_ops_calc_mem_size(mp, mp->size, pg_shift,  in get_anon_size()
640 	size = get_anon_size(memhdr->mp);  in rte_mempool_memchunk_anon_free()
649 rte_mempool_populate_anon(struct rte_mempool *mp)  in rte_mempool_populate_anon()  argument
656 	if ((!STAILQ_EMPTY(&mp->mem_list)) || mp->nb_mem_chunks != 0) {  in rte_mempool_populate_anon()
661 	ret = mempool_ops_alloc_once(mp);  in rte_mempool_populate_anon()
667 	size = get_anon_size(mp);  in rte_mempool_populate_anon()
684 	ret = rte_mempool_populate_virt(mp, addr, size, rte_mem_page_size(),  in rte_mempool_populate_anon()
693 	rte_mempool_trace_populate_anon(mp);  in rte_mempool_populate_anon()
694 	return mp->populated_size;  in rte_mempool_populate_anon()
697 	rte_mempool_free_memchunks(mp);  in rte_mempool_populate_anon()
703 rte_mempool_free(struct rte_mempool *mp)  in rte_mempool_free()  argument
708 	if (mp == NULL)  in rte_mempool_free()
715 		if (te->data == (void *)mp)  in rte_mempool_free()
725 	rte_mempool_trace_free(mp);  in rte_mempool_free()
726 	rte_mempool_free_memchunks(mp);  in rte_mempool_free()
727 	rte_mempool_ops_free(mp);  in rte_mempool_free()
728 	rte_memzone_free(mp->mz);  in rte_mempool_free()
788 	struct rte_mempool *mp = NULL;  in rte_mempool_create_empty()  local
851 	mempool_size = MEMPOOL_HEADER_SIZE(mp, cache_size);  in rte_mempool_create_empty()
866 	mp = mz->addr;  in rte_mempool_create_empty()
867 	memset(mp, 0, MEMPOOL_HEADER_SIZE(mp, cache_size));  in rte_mempool_create_empty()
868 	ret = strlcpy(mp->name, name, sizeof(mp->name));  in rte_mempool_create_empty()
869 	if (ret < 0 || ret >= (int)sizeof(mp->name)) {  in rte_mempool_create_empty()
873 	mp->mz = mz;  in rte_mempool_create_empty()
874 	mp->size = n;  in rte_mempool_create_empty()
875 	mp->flags = flags;  in rte_mempool_create_empty()
876 	mp->socket_id = socket_id;  in rte_mempool_create_empty()
877 	mp->elt_size = objsz.elt_size;  in rte_mempool_create_empty()
878 	mp->header_size = objsz.header_size;  in rte_mempool_create_empty()
879 	mp->trailer_size = objsz.trailer_size;  in rte_mempool_create_empty()
881 	mp->cache_size = cache_size;  in rte_mempool_create_empty()
882 	mp->private_data_size = private_data_size;  in rte_mempool_create_empty()
883 	STAILQ_INIT(&mp->elt_list);  in rte_mempool_create_empty()
884 	STAILQ_INIT(&mp->mem_list);  in rte_mempool_create_empty()
890 	mp->local_cache = (struct rte_mempool_cache *)  in rte_mempool_create_empty()
891 		RTE_PTR_ADD(mp, MEMPOOL_HEADER_SIZE(mp, 0));  in rte_mempool_create_empty()
896 			mempool_cache_init(&mp->local_cache[lcore_id],  in rte_mempool_create_empty()
900 	te->data = mp;  in rte_mempool_create_empty()
908 		private_data_size, flags, mp);  in rte_mempool_create_empty()
909 	return mp;  in rte_mempool_create_empty()
914 	rte_mempool_free(mp);  in rte_mempool_create_empty()
927 	struct rte_mempool *mp;  in rte_mempool_create()  local
929 	mp = rte_mempool_create_empty(name, n, elt_size, cache_size,  in rte_mempool_create()
931 	if (mp == NULL)  in rte_mempool_create()
939 		ret = rte_mempool_set_ops_byname(mp, "ring_sp_sc", NULL);  in rte_mempool_create()
941 		ret = rte_mempool_set_ops_byname(mp, "ring_sp_mc", NULL);  in rte_mempool_create()
943 		ret = rte_mempool_set_ops_byname(mp, "ring_mp_sc", NULL);  in rte_mempool_create()
945 		ret = rte_mempool_set_ops_byname(mp, "ring_mp_mc", NULL);  in rte_mempool_create()
952 		mp_init(mp, mp_init_arg);  in rte_mempool_create()
954 	if (rte_mempool_populate_default(mp) < 0)  in rte_mempool_create()
959 		rte_mempool_obj_iter(mp, obj_init, obj_init_arg);  in rte_mempool_create()
963 		obj_init_arg, flags, mp);  in rte_mempool_create()
964 	return mp;  in rte_mempool_create()
967 	rte_mempool_free(mp);  in rte_mempool_create()
973 rte_mempool_avail_count(const struct rte_mempool *mp)  in rte_mempool_avail_count()  argument
978 	count = rte_mempool_ops_get_count(mp);  in rte_mempool_avail_count()
980 	if (mp->cache_size == 0)  in rte_mempool_avail_count()
984 		count += mp->local_cache[lcore_id].len;  in rte_mempool_avail_count()
990 	if (count > mp->size)  in rte_mempool_avail_count()
991 		return mp->size;  in rte_mempool_avail_count()
997 rte_mempool_in_use_count(const struct rte_mempool *mp)  in rte_mempool_in_use_count()  argument
999 	return mp->size - rte_mempool_avail_count(mp);  in rte_mempool_in_use_count()
1004 rte_mempool_dump_cache(FILE *f, const struct rte_mempool *mp)  in rte_mempool_dump_cache()  argument
1011 	fprintf(f, "    cache_size=%"PRIu32"\n", mp->cache_size);  in rte_mempool_dump_cache()
1013 	if (mp->cache_size == 0)  in rte_mempool_dump_cache()
1017 		cache_count = mp->local_cache[lcore_id].len;  in rte_mempool_dump_cache()
1031 void rte_mempool_check_cookies(const struct rte_mempool *mp,  in rte_mempool_check_cookies()  argument
1050 		if (rte_mempool_from_obj(obj) != mp)  in rte_mempool_check_cookies()
1061 					obj, (const void *) mp, cookie);  in rte_mempool_check_cookies()
1069 					obj, (const void *) mp, cookie);  in rte_mempool_check_cookies()
1078 					obj, (const void *) mp, cookie);  in rte_mempool_check_cookies()
1087 				obj, (const void *) mp, cookie);  in rte_mempool_check_cookies()
1092 	RTE_SET_USED(mp);  in rte_mempool_check_cookies()
1100 rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,  in rte_mempool_contig_blocks_check_cookies()  argument
1106 		mp->header_size + mp->elt_size + mp->trailer_size;  in rte_mempool_contig_blocks_check_cookies()
1109 	rte_mempool_ops_get_info(mp, &info);  in rte_mempool_contig_blocks_check_cookies()
1118 			rte_mempool_check_cookies(mp, &obj, 1, free);  in rte_mempool_contig_blocks_check_cookies()
1122 	RTE_SET_USED(mp);  in rte_mempool_contig_blocks_check_cookies()
1131 mempool_obj_audit(struct rte_mempool *mp, __rte_unused void *opaque,  in mempool_obj_audit()  argument
1134 	__mempool_check_cookies(mp, &obj, 1, 2);  in mempool_obj_audit()
1138 mempool_audit_cookies(struct rte_mempool *mp)  in mempool_audit_cookies()  argument
1142 	num = rte_mempool_obj_iter(mp, mempool_obj_audit, NULL);  in mempool_audit_cookies()
1143 	if (num != mp->size) {  in mempool_audit_cookies()
1146 			mp, mp->size, num);  in mempool_audit_cookies()
1150 #define mempool_audit_cookies(mp) do {} while(0)  argument
1159 mempool_audit_cache(const struct rte_mempool *mp)  in mempool_audit_cache()  argument
1164 	if (mp->cache_size == 0)  in mempool_audit_cache()
1169 		cache = &mp->local_cache[lcore_id];  in mempool_audit_cache()
1180 rte_mempool_audit(struct rte_mempool *mp)  in rte_mempool_audit()  argument
1182 	mempool_audit_cache(mp);  in rte_mempool_audit()
1183 	mempool_audit_cookies(mp);  in rte_mempool_audit()
1186 	RTE_SET_USED(mp);  in rte_mempool_audit()
1191 rte_mempool_dump(FILE *f, struct rte_mempool *mp)  in rte_mempool_dump()  argument
1205 	RTE_ASSERT(mp != NULL);  in rte_mempool_dump()
1207 	fprintf(f, "mempool <%s>@%p\n", mp->name, mp);  in rte_mempool_dump()
1208 	fprintf(f, "  flags=%x\n", mp->flags);  in rte_mempool_dump()
1209 	fprintf(f, "  socket_id=%d\n", mp->socket_id);  in rte_mempool_dump()
1210 	fprintf(f, "  pool=%p\n", mp->pool_data);  in rte_mempool_dump()
1211 	fprintf(f, "  iova=0x%" PRIx64 "\n", mp->mz->iova);  in rte_mempool_dump()
1212 	fprintf(f, "  nb_mem_chunks=%u\n", mp->nb_mem_chunks);  in rte_mempool_dump()
1213 	fprintf(f, "  size=%"PRIu32"\n", mp->size);  in rte_mempool_dump()
1214 	fprintf(f, "  populated_size=%"PRIu32"\n", mp->populated_size);  in rte_mempool_dump()
1215 	fprintf(f, "  header_size=%"PRIu32"\n", mp->header_size);  in rte_mempool_dump()
1216 	fprintf(f, "  elt_size=%"PRIu32"\n", mp->elt_size);  in rte_mempool_dump()
1217 	fprintf(f, "  trailer_size=%"PRIu32"\n", mp->trailer_size);  in rte_mempool_dump()
1219 	       mp->header_size + mp->elt_size + mp->trailer_size);  in rte_mempool_dump()
1221 	fprintf(f, "  private_data_size=%"PRIu32"\n", mp->private_data_size);  in rte_mempool_dump()
1223 	fprintf(f, "  ops_index=%d\n", mp->ops_index);  in rte_mempool_dump()
1224 	ops = rte_mempool_get_ops(mp->ops_index);  in rte_mempool_dump()
1227 	STAILQ_FOREACH(memhdr, &mp->mem_list, next)  in rte_mempool_dump()
1231 			(long double)mem_len / mp->size);  in rte_mempool_dump()
1234 	cache_count = rte_mempool_dump_cache(f, mp);  in rte_mempool_dump()
1235 	common_count = rte_mempool_ops_get_count(mp);  in rte_mempool_dump()
1236 	if ((cache_count + common_count) > mp->size)  in rte_mempool_dump()
1237 		common_count = mp->size - cache_count;  in rte_mempool_dump()
1242 	rte_mempool_ops_get_info(mp, &info);  in rte_mempool_dump()
1245 		sum.put_bulk += mp->stats[lcore_id].put_bulk;  in rte_mempool_dump()
1246 		sum.put_objs += mp->stats[lcore_id].put_objs;  in rte_mempool_dump()
1247 		sum.get_success_bulk += mp->stats[lcore_id].get_success_bulk;  in rte_mempool_dump()
1248 		sum.get_success_objs += mp->stats[lcore_id].get_success_objs;  in rte_mempool_dump()
1249 		sum.get_fail_bulk += mp->stats[lcore_id].get_fail_bulk;  in rte_mempool_dump()
1250 		sum.get_fail_objs += mp->stats[lcore_id].get_fail_objs;  in rte_mempool_dump()
1251 		sum.get_success_blks += mp->stats[lcore_id].get_success_blks;  in rte_mempool_dump()
1252 		sum.get_fail_blks += mp->stats[lcore_id].get_fail_blks;  in rte_mempool_dump()
1270 	rte_mempool_audit(mp);  in rte_mempool_dump()
1277 	struct rte_mempool *mp = NULL;  in rte_mempool_list_dump()  local
1286 		mp = (struct rte_mempool *) te->data;  in rte_mempool_list_dump()
1287 		rte_mempool_dump(f, mp);  in rte_mempool_list_dump()
1297 	struct rte_mempool *mp = NULL;  in rte_mempool_lookup()  local
1306 		mp = (struct rte_mempool *) te->data;  in rte_mempool_lookup()
1307 		if (strncmp(name, mp->name, RTE_MEMPOOL_NAMESIZE) == 0)  in rte_mempool_lookup()
1318 	return mp;  in rte_mempool_lookup()