Lines Matching refs:pool
383 mlx5_ipool_lock(struct mlx5_indexed_pool *pool) in mlx5_ipool_lock() argument
385 if (pool->cfg.need_lock) in mlx5_ipool_lock()
386 rte_spinlock_lock(&pool->lock); in mlx5_ipool_lock()
390 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool) in mlx5_ipool_unlock() argument
392 if (pool->cfg.need_lock) in mlx5_ipool_unlock()
393 rte_spinlock_unlock(&pool->lock); in mlx5_ipool_unlock()
397 mlx5_trunk_idx_get(struct mlx5_indexed_pool *pool, uint32_t entry_idx) in mlx5_trunk_idx_get() argument
399 struct mlx5_indexed_pool_config *cfg = &pool->cfg; in mlx5_trunk_idx_get()
405 if (entry_idx >= pool->grow_tbl[cfg->grow_trunk - 1]) { in mlx5_trunk_idx_get()
406 trunk_idx = (entry_idx - pool->grow_tbl[cfg->grow_trunk - 1]) / in mlx5_trunk_idx_get()
411 if (entry_idx < pool->grow_tbl[i]) in mlx5_trunk_idx_get()
420 mlx5_trunk_size_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx) in mlx5_trunk_size_get() argument
422 struct mlx5_indexed_pool_config *cfg = &pool->cfg; in mlx5_trunk_size_get()
429 mlx5_trunk_idx_offset_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx) in mlx5_trunk_idx_offset_get() argument
431 struct mlx5_indexed_pool_config *cfg = &pool->cfg; in mlx5_trunk_idx_offset_get()
439 offset = pool->grow_tbl[trunk_idx - 1]; in mlx5_trunk_idx_offset_get()
441 offset = pool->grow_tbl[cfg->grow_trunk - 1] + in mlx5_trunk_idx_offset_get()
450 struct mlx5_indexed_pool *pool; in mlx5_ipool_create() local
457 pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool) + cfg->grow_trunk * in mlx5_ipool_create()
458 sizeof(pool->grow_tbl[0]), RTE_CACHE_LINE_SIZE, in mlx5_ipool_create()
460 if (!pool) in mlx5_ipool_create()
462 pool->cfg = *cfg; in mlx5_ipool_create()
463 if (!pool->cfg.trunk_size) in mlx5_ipool_create()
464 pool->cfg.trunk_size = MLX5_IPOOL_DEFAULT_TRUNK_SIZE; in mlx5_ipool_create()
466 pool->cfg.malloc = mlx5_malloc; in mlx5_ipool_create()
467 pool->cfg.free = mlx5_free; in mlx5_ipool_create()
469 pool->free_list = TRUNK_INVALID; in mlx5_ipool_create()
470 if (pool->cfg.need_lock) in mlx5_ipool_create()
471 rte_spinlock_init(&pool->lock); in mlx5_ipool_create()
477 pool->grow_tbl[i] = cfg->trunk_size << (cfg->grow_shift * i); in mlx5_ipool_create()
479 pool->grow_tbl[i] += pool->grow_tbl[i - 1]; in mlx5_ipool_create()
481 return pool; in mlx5_ipool_create()
485 mlx5_ipool_grow(struct mlx5_indexed_pool *pool) in mlx5_ipool_grow() argument
495 if (pool->n_trunk_valid == TRUNK_MAX_IDX) in mlx5_ipool_grow()
497 if (pool->n_trunk_valid == pool->n_trunk) { in mlx5_ipool_grow()
499 int n_grow = pool->n_trunk_valid ? pool->n_trunk : in mlx5_ipool_grow()
502 p = pool->cfg.malloc(0, (pool->n_trunk_valid + n_grow) * in mlx5_ipool_grow()
507 if (pool->trunks) in mlx5_ipool_grow()
508 memcpy(p, pool->trunks, pool->n_trunk_valid * in mlx5_ipool_grow()
510 memset(RTE_PTR_ADD(p, pool->n_trunk_valid * sizeof(void *)), 0, in mlx5_ipool_grow()
512 trunk_tmp = pool->trunks; in mlx5_ipool_grow()
513 pool->trunks = p; in mlx5_ipool_grow()
515 pool->cfg.free(trunk_tmp); in mlx5_ipool_grow()
516 pool->n_trunk += n_grow; in mlx5_ipool_grow()
518 if (!pool->cfg.release_mem_en) { in mlx5_ipool_grow()
519 idx = pool->n_trunk_valid; in mlx5_ipool_grow()
522 for (idx = 0; idx < pool->n_trunk; idx++) in mlx5_ipool_grow()
523 if (pool->trunks[idx] == NULL) in mlx5_ipool_grow()
527 data_size = mlx5_trunk_size_get(pool, idx); in mlx5_ipool_grow()
530 trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size); in mlx5_ipool_grow()
532 trunk = pool->cfg.malloc(0, trunk_size, in mlx5_ipool_grow()
536 pool->trunks[idx] = trunk; in mlx5_ipool_grow()
541 MLX5_ASSERT(pool->free_list == TRUNK_INVALID); in mlx5_ipool_grow()
542 pool->free_list = idx; in mlx5_ipool_grow()
545 [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)], in mlx5_ipool_grow()
548 pool->n_trunk_valid++; in mlx5_ipool_grow()
550 pool->trunk_new++; in mlx5_ipool_grow()
551 pool->trunk_avail++; in mlx5_ipool_grow()
557 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx) in mlx5_ipool_malloc() argument
564 mlx5_ipool_lock(pool); in mlx5_ipool_malloc()
565 if (pool->free_list == TRUNK_INVALID) { in mlx5_ipool_malloc()
567 if (mlx5_ipool_grow(pool)) { in mlx5_ipool_malloc()
568 mlx5_ipool_unlock(pool); in mlx5_ipool_malloc()
572 MLX5_ASSERT(pool->free_list != TRUNK_INVALID); in mlx5_ipool_malloc()
573 trunk = pool->trunks[pool->free_list]; in mlx5_ipool_malloc()
576 mlx5_ipool_unlock(pool); in mlx5_ipool_malloc()
582 MLX5_ASSERT(iidx < mlx5_trunk_size_get(pool, trunk->idx)); in mlx5_ipool_malloc()
584 p = &trunk->data[iidx * pool->cfg.size]; in mlx5_ipool_malloc()
590 iidx += mlx5_trunk_idx_offset_get(pool, trunk->idx); in mlx5_ipool_malloc()
594 pool->n_entry++; in mlx5_ipool_malloc()
598 MLX5_ASSERT(pool->free_list == trunk->idx); in mlx5_ipool_malloc()
599 pool->free_list = trunk->next; in mlx5_ipool_malloc()
601 pool->trunks[trunk->next]->prev = TRUNK_INVALID; in mlx5_ipool_malloc()
605 pool->trunk_empty++; in mlx5_ipool_malloc()
606 pool->trunk_avail--; in mlx5_ipool_malloc()
610 mlx5_ipool_unlock(pool); in mlx5_ipool_malloc()
615 mlx5_ipool_zmalloc(struct mlx5_indexed_pool *pool, uint32_t *idx) in mlx5_ipool_zmalloc() argument
617 void *entry = mlx5_ipool_malloc(pool, idx); in mlx5_ipool_zmalloc()
619 if (entry && pool->cfg.size) in mlx5_ipool_zmalloc()
620 memset(entry, 0, pool->cfg.size); in mlx5_ipool_zmalloc()
625 mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx) in mlx5_ipool_free() argument
634 mlx5_ipool_lock(pool); in mlx5_ipool_free()
635 trunk_idx = mlx5_trunk_idx_get(pool, idx); in mlx5_ipool_free()
636 if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) || in mlx5_ipool_free()
637 (pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk)) in mlx5_ipool_free()
639 trunk = pool->trunks[trunk_idx]; in mlx5_ipool_free()
642 entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx); in mlx5_ipool_free()
648 if (pool->cfg.release_mem_en && trunk->free == mlx5_trunk_size_get in mlx5_ipool_free()
649 (pool, trunk->idx)) { in mlx5_ipool_free()
650 if (pool->free_list == trunk->idx) in mlx5_ipool_free()
651 pool->free_list = trunk->next; in mlx5_ipool_free()
653 pool->trunks[trunk->next]->prev = trunk->prev; in mlx5_ipool_free()
655 pool->trunks[trunk->prev]->next = trunk->next; in mlx5_ipool_free()
656 pool->cfg.free(trunk); in mlx5_ipool_free()
657 pool->trunks[trunk_idx] = NULL; in mlx5_ipool_free()
658 pool->n_trunk_valid--; in mlx5_ipool_free()
660 pool->trunk_avail--; in mlx5_ipool_free()
661 pool->trunk_free++; in mlx5_ipool_free()
663 if (pool->n_trunk_valid == 0) { in mlx5_ipool_free()
664 pool->cfg.free(pool->trunks); in mlx5_ipool_free()
665 pool->trunks = NULL; in mlx5_ipool_free()
666 pool->n_trunk = 0; in mlx5_ipool_free()
670 MLX5_ASSERT(pool->free_list != trunk->idx); in mlx5_ipool_free()
671 trunk->next = pool->free_list; in mlx5_ipool_free()
673 if (pool->free_list != TRUNK_INVALID) in mlx5_ipool_free()
674 pool->trunks[pool->free_list]->prev = trunk->idx; in mlx5_ipool_free()
675 pool->free_list = trunk->idx; in mlx5_ipool_free()
677 pool->trunk_empty--; in mlx5_ipool_free()
678 pool->trunk_avail++; in mlx5_ipool_free()
682 pool->n_entry--; in mlx5_ipool_free()
685 mlx5_ipool_unlock(pool); in mlx5_ipool_free()
689 mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx) in mlx5_ipool_get() argument
699 mlx5_ipool_lock(pool); in mlx5_ipool_get()
700 trunk_idx = mlx5_trunk_idx_get(pool, idx); in mlx5_ipool_get()
701 if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) || in mlx5_ipool_get()
702 (pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk)) in mlx5_ipool_get()
704 trunk = pool->trunks[trunk_idx]; in mlx5_ipool_get()
707 entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx); in mlx5_ipool_get()
711 p = &trunk->data[entry_idx * pool->cfg.size]; in mlx5_ipool_get()
713 mlx5_ipool_unlock(pool); in mlx5_ipool_get()
718 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool) in mlx5_ipool_destroy() argument
723 MLX5_ASSERT(pool); in mlx5_ipool_destroy()
724 mlx5_ipool_lock(pool); in mlx5_ipool_destroy()
725 trunks = pool->trunks; in mlx5_ipool_destroy()
726 for (i = 0; i < pool->n_trunk; i++) { in mlx5_ipool_destroy()
728 pool->cfg.free(trunks[i]); in mlx5_ipool_destroy()
730 if (!pool->trunks) in mlx5_ipool_destroy()
731 pool->cfg.free(pool->trunks); in mlx5_ipool_destroy()
732 mlx5_ipool_unlock(pool); in mlx5_ipool_destroy()
733 mlx5_free(pool); in mlx5_ipool_destroy()
738 mlx5_ipool_dump(struct mlx5_indexed_pool *pool) in mlx5_ipool_dump() argument
742 pool->cfg.type, pool->cfg.size, pool->n_trunk_valid, in mlx5_ipool_dump()
743 pool->cfg.trunk_size, pool->n_trunk_valid); in mlx5_ipool_dump()
747 pool->cfg.type, pool->n_entry, pool->trunk_new, in mlx5_ipool_dump()
748 pool->trunk_empty, pool->trunk_avail, pool->trunk_free); in mlx5_ipool_dump()