1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright 2017 NXP 4 * 5 */ 6 7 /* System headers */ 8 #include <stdio.h> 9 #include <inttypes.h> 10 #include <unistd.h> 11 #include <limits.h> 12 #include <sched.h> 13 #include <signal.h> 14 #include <pthread.h> 15 #include <sys/types.h> 16 #include <sys/syscall.h> 17 18 #include <rte_byteorder.h> 19 #include <rte_common.h> 20 #include <rte_log.h> 21 #include <rte_debug.h> 22 #include <rte_memory.h> 23 #include <rte_tailq.h> 24 #include <rte_eal.h> 25 #include <rte_malloc.h> 26 #include <rte_ring.h> 27 28 #include <dpaa_mempool.h> 29 30 struct dpaa_bp_info rte_dpaa_bpid_info[DPAA_MAX_BPOOLS]; 31 32 static int 33 dpaa_mbuf_create_pool(struct rte_mempool *mp) 34 { 35 struct bman_pool *bp; 36 struct bm_buffer bufs[8]; 37 struct dpaa_bp_info *bp_info; 38 uint8_t bpid; 39 int num_bufs = 0, ret = 0; 40 struct bman_pool_params params = { 41 .flags = BMAN_POOL_FLAG_DYNAMIC_BPID 42 }; 43 44 MEMPOOL_INIT_FUNC_TRACE(); 45 46 bp = bman_new_pool(¶ms); 47 if (!bp) { 48 DPAA_MEMPOOL_ERR("bman_new_pool() failed"); 49 return -ENODEV; 50 } 51 bpid = bman_get_params(bp)->bpid; 52 53 /* Drain the pool of anything already in it. */ 54 do { 55 /* Acquire is all-or-nothing, so we drain in 8s, 56 * then in 1s for the remainder. 57 */ 58 if (ret != 1) 59 ret = bman_acquire(bp, bufs, 8, 0); 60 if (ret < 8) 61 ret = bman_acquire(bp, bufs, 1, 0); 62 if (ret > 0) 63 num_bufs += ret; 64 } while (ret > 0); 65 if (num_bufs) 66 DPAA_MEMPOOL_WARN("drained %u bufs from BPID %d", 67 num_bufs, bpid); 68 69 rte_dpaa_bpid_info[bpid].mp = mp; 70 rte_dpaa_bpid_info[bpid].bpid = bpid; 71 rte_dpaa_bpid_info[bpid].size = mp->elt_size; 72 rte_dpaa_bpid_info[bpid].bp = bp; 73 rte_dpaa_bpid_info[bpid].meta_data_size = 74 sizeof(struct rte_mbuf) + rte_pktmbuf_priv_size(mp); 75 rte_dpaa_bpid_info[bpid].dpaa_ops_index = mp->ops_index; 76 rte_dpaa_bpid_info[bpid].ptov_off = 0; 77 rte_dpaa_bpid_info[bpid].flags = 0; 78 79 bp_info = rte_malloc(NULL, 80 sizeof(struct dpaa_bp_info), 81 RTE_CACHE_LINE_SIZE); 82 if (!bp_info) { 83 DPAA_MEMPOOL_WARN("Memory allocation failed for bp_info"); 84 bman_free_pool(bp); 85 return -ENOMEM; 86 } 87 88 rte_memcpy(bp_info, (void *)&rte_dpaa_bpid_info[bpid], 89 sizeof(struct dpaa_bp_info)); 90 mp->pool_data = (void *)bp_info; 91 92 DPAA_MEMPOOL_INFO("BMAN pool created for bpid =%d", bpid); 93 return 0; 94 } 95 96 static void 97 dpaa_mbuf_free_pool(struct rte_mempool *mp) 98 { 99 struct dpaa_bp_info *bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 100 101 MEMPOOL_INIT_FUNC_TRACE(); 102 103 if (bp_info) { 104 bman_free_pool(bp_info->bp); 105 DPAA_MEMPOOL_INFO("BMAN pool freed for bpid =%d", 106 bp_info->bpid); 107 rte_free(mp->pool_data); 108 mp->pool_data = NULL; 109 } 110 } 111 112 static void 113 dpaa_buf_free(struct dpaa_bp_info *bp_info, uint64_t addr) 114 { 115 struct bm_buffer buf; 116 int ret; 117 118 DPAA_MEMPOOL_DEBUG("Free 0x%lx to bpid: %d", addr, bp_info->bpid); 119 120 bm_buffer_set64(&buf, addr); 121 retry: 122 ret = bman_release(bp_info->bp, &buf, 1, 0); 123 if (ret) { 124 DPAA_MEMPOOL_DEBUG("BMAN busy. Retrying..."); 125 cpu_spin(CPU_SPIN_BACKOFF_CYCLES); 126 goto retry; 127 } 128 } 129 130 static int 131 dpaa_mbuf_free_bulk(struct rte_mempool *pool, 132 void *const *obj_table, 133 unsigned int n) 134 { 135 struct dpaa_bp_info *bp_info = DPAA_MEMPOOL_TO_POOL_INFO(pool); 136 int ret; 137 unsigned int i = 0; 138 139 DPAA_MEMPOOL_DPDEBUG("Request to free %d buffers in bpid = %d", 140 n, bp_info->bpid); 141 142 ret = rte_dpaa_portal_init((void *)0); 143 if (ret) { 144 DPAA_MEMPOOL_ERR("rte_dpaa_portal_init failed with ret: %d", 145 ret); 146 return 0; 147 } 148 149 while (i < n) { 150 uint64_t phy = rte_mempool_virt2iova(obj_table[i]); 151 152 if (unlikely(!bp_info->ptov_off)) { 153 /* buffers are not from multiple memzones */ 154 if (!(bp_info->flags & DPAA_MPOOL_MULTI_MEMZONE)) { 155 bp_info->ptov_off 156 = (uint64_t)obj_table[i] - phy; 157 rte_dpaa_bpid_info[bp_info->bpid].ptov_off 158 = bp_info->ptov_off; 159 } 160 } 161 162 dpaa_buf_free(bp_info, 163 (uint64_t)phy + bp_info->meta_data_size); 164 i = i + 1; 165 } 166 167 DPAA_MEMPOOL_DPDEBUG("freed %d buffers in bpid =%d", 168 n, bp_info->bpid); 169 170 return 0; 171 } 172 173 static int 174 dpaa_mbuf_alloc_bulk(struct rte_mempool *pool, 175 void **obj_table, 176 unsigned int count) 177 { 178 struct rte_mbuf **m = (struct rte_mbuf **)obj_table; 179 struct bm_buffer bufs[DPAA_MBUF_MAX_ACQ_REL]; 180 struct dpaa_bp_info *bp_info; 181 void *bufaddr; 182 int i, ret; 183 unsigned int n = 0; 184 185 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(pool); 186 187 DPAA_MEMPOOL_DPDEBUG("Request to alloc %d buffers in bpid = %d", 188 count, bp_info->bpid); 189 190 if (unlikely(count >= (RTE_MEMPOOL_CACHE_MAX_SIZE * 2))) { 191 DPAA_MEMPOOL_ERR("Unable to allocate requested (%u) buffers", 192 count); 193 return -1; 194 } 195 196 ret = rte_dpaa_portal_init((void *)0); 197 if (ret) { 198 DPAA_MEMPOOL_ERR("rte_dpaa_portal_init failed with ret: %d", 199 ret); 200 return -1; 201 } 202 203 while (n < count) { 204 /* Acquire is all-or-nothing, so we drain in 7s, 205 * then the remainder. 206 */ 207 if ((count - n) > DPAA_MBUF_MAX_ACQ_REL) { 208 ret = bman_acquire(bp_info->bp, bufs, 209 DPAA_MBUF_MAX_ACQ_REL, 0); 210 } else { 211 ret = bman_acquire(bp_info->bp, bufs, count - n, 0); 212 } 213 /* In case of less than requested number of buffers available 214 * in pool, qbman_swp_acquire returns 0 215 */ 216 if (ret <= 0) { 217 DPAA_MEMPOOL_DPDEBUG("Buffer acquire failed (%d)", 218 ret); 219 /* The API expect the exact number of requested 220 * buffers. Releasing all buffers allocated 221 */ 222 dpaa_mbuf_free_bulk(pool, obj_table, n); 223 return -ENOBUFS; 224 } 225 /* assigning mbuf from the acquired objects */ 226 for (i = 0; (i < ret) && bufs[i].addr; i++) { 227 /* TODO-errata - objerved that bufs may be null 228 * i.e. first buffer is valid, remaining 6 buffers 229 * may be null. 230 */ 231 bufaddr = DPAA_MEMPOOL_PTOV(bp_info, bufs[i].addr); 232 m[n] = (struct rte_mbuf *)((char *)bufaddr 233 - bp_info->meta_data_size); 234 DPAA_MEMPOOL_DPDEBUG("Paddr (%p), FD (%p) from BMAN", 235 (void *)bufaddr, (void *)m[n]); 236 n++; 237 } 238 } 239 240 DPAA_MEMPOOL_DPDEBUG("Allocated %d buffers from bpid=%d", 241 n, bp_info->bpid); 242 return 0; 243 } 244 245 static unsigned int 246 dpaa_mbuf_get_count(const struct rte_mempool *mp) 247 { 248 struct dpaa_bp_info *bp_info; 249 250 MEMPOOL_INIT_FUNC_TRACE(); 251 252 if (!mp || !mp->pool_data) { 253 DPAA_MEMPOOL_ERR("Invalid mempool provided\n"); 254 return 0; 255 } 256 257 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 258 259 return bman_query_free_buffers(bp_info->bp); 260 } 261 262 static int 263 dpaa_register_memory_area(const struct rte_mempool *mp, 264 char *vaddr __rte_unused, 265 rte_iova_t paddr __rte_unused, 266 size_t len) 267 { 268 struct dpaa_bp_info *bp_info; 269 unsigned int total_elt_sz; 270 271 MEMPOOL_INIT_FUNC_TRACE(); 272 273 if (!mp || !mp->pool_data) { 274 DPAA_MEMPOOL_ERR("Invalid mempool provided\n"); 275 return 0; 276 } 277 278 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 279 total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; 280 281 DPAA_MEMPOOL_DEBUG("Req size %lu vs Available %u\n", 282 len, total_elt_sz * mp->size); 283 284 /* Detect pool area has sufficient space for elements in this memzone */ 285 if (len < total_elt_sz * mp->size) 286 /* Else, Memory will be allocated from multiple memzones */ 287 bp_info->flags |= DPAA_MPOOL_MULTI_MEMZONE; 288 289 return 0; 290 } 291 292 struct rte_mempool_ops dpaa_mpool_ops = { 293 .name = "dpaa", 294 .alloc = dpaa_mbuf_create_pool, 295 .free = dpaa_mbuf_free_pool, 296 .enqueue = dpaa_mbuf_free_bulk, 297 .dequeue = dpaa_mbuf_alloc_bulk, 298 .get_count = dpaa_mbuf_get_count, 299 .register_memory_area = dpaa_register_memory_area, 300 }; 301 302 MEMPOOL_REGISTER_OPS(dpaa_mpool_ops); 303