1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright 2017 NXP 4 * 5 */ 6 7 /* System headers */ 8 #include <stdio.h> 9 #include <inttypes.h> 10 #include <unistd.h> 11 #include <limits.h> 12 #include <sched.h> 13 #include <signal.h> 14 #include <pthread.h> 15 #include <sys/types.h> 16 #include <sys/syscall.h> 17 18 #include <rte_byteorder.h> 19 #include <rte_common.h> 20 #include <rte_log.h> 21 #include <rte_debug.h> 22 #include <rte_memory.h> 23 #include <rte_tailq.h> 24 #include <rte_eal.h> 25 #include <rte_malloc.h> 26 #include <rte_ring.h> 27 28 #include <dpaa_mempool.h> 29 30 struct dpaa_bp_info rte_dpaa_bpid_info[DPAA_MAX_BPOOLS]; 31 32 static int 33 dpaa_mbuf_create_pool(struct rte_mempool *mp) 34 { 35 struct bman_pool *bp; 36 struct bm_buffer bufs[8]; 37 struct dpaa_bp_info *bp_info; 38 uint8_t bpid; 39 int num_bufs = 0, ret = 0; 40 struct bman_pool_params params = { 41 .flags = BMAN_POOL_FLAG_DYNAMIC_BPID 42 }; 43 44 MEMPOOL_INIT_FUNC_TRACE(); 45 46 bp = bman_new_pool(¶ms); 47 if (!bp) { 48 DPAA_MEMPOOL_ERR("bman_new_pool() failed"); 49 return -ENODEV; 50 } 51 bpid = bman_get_params(bp)->bpid; 52 53 /* Drain the pool of anything already in it. */ 54 do { 55 /* Acquire is all-or-nothing, so we drain in 8s, 56 * then in 1s for the remainder. 57 */ 58 if (ret != 1) 59 ret = bman_acquire(bp, bufs, 8, 0); 60 if (ret < 8) 61 ret = bman_acquire(bp, bufs, 1, 0); 62 if (ret > 0) 63 num_bufs += ret; 64 } while (ret > 0); 65 if (num_bufs) 66 DPAA_MEMPOOL_WARN("drained %u bufs from BPID %d", 67 num_bufs, bpid); 68 69 rte_dpaa_bpid_info[bpid].mp = mp; 70 rte_dpaa_bpid_info[bpid].bpid = bpid; 71 rte_dpaa_bpid_info[bpid].size = mp->elt_size; 72 rte_dpaa_bpid_info[bpid].bp = bp; 73 rte_dpaa_bpid_info[bpid].meta_data_size = 74 sizeof(struct rte_mbuf) + rte_pktmbuf_priv_size(mp); 75 rte_dpaa_bpid_info[bpid].dpaa_ops_index = mp->ops_index; 76 rte_dpaa_bpid_info[bpid].ptov_off = 0; 77 rte_dpaa_bpid_info[bpid].flags = 0; 78 79 bp_info = rte_malloc(NULL, 80 sizeof(struct dpaa_bp_info), 81 RTE_CACHE_LINE_SIZE); 82 if (!bp_info) { 83 DPAA_MEMPOOL_WARN("Memory allocation failed for bp_info"); 84 bman_free_pool(bp); 85 return -ENOMEM; 86 } 87 88 rte_memcpy(bp_info, (void *)&rte_dpaa_bpid_info[bpid], 89 sizeof(struct dpaa_bp_info)); 90 mp->pool_data = (void *)bp_info; 91 92 DPAA_MEMPOOL_INFO("BMAN pool created for bpid =%d", bpid); 93 return 0; 94 } 95 96 static void 97 dpaa_mbuf_free_pool(struct rte_mempool *mp) 98 { 99 struct dpaa_bp_info *bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 100 101 MEMPOOL_INIT_FUNC_TRACE(); 102 103 if (bp_info) { 104 bman_free_pool(bp_info->bp); 105 DPAA_MEMPOOL_INFO("BMAN pool freed for bpid =%d", 106 bp_info->bpid); 107 rte_free(mp->pool_data); 108 mp->pool_data = NULL; 109 } 110 } 111 112 static void 113 dpaa_buf_free(struct dpaa_bp_info *bp_info, uint64_t addr) 114 { 115 struct bm_buffer buf; 116 int ret; 117 118 DPAA_MEMPOOL_DEBUG("Free 0x%" PRIx64 " to bpid: %d", 119 addr, bp_info->bpid); 120 121 bm_buffer_set64(&buf, addr); 122 retry: 123 ret = bman_release(bp_info->bp, &buf, 1, 0); 124 if (ret) { 125 DPAA_MEMPOOL_DEBUG("BMAN busy. Retrying..."); 126 cpu_spin(CPU_SPIN_BACKOFF_CYCLES); 127 goto retry; 128 } 129 } 130 131 static int 132 dpaa_mbuf_free_bulk(struct rte_mempool *pool, 133 void *const *obj_table, 134 unsigned int n) 135 { 136 struct dpaa_bp_info *bp_info = DPAA_MEMPOOL_TO_POOL_INFO(pool); 137 int ret; 138 unsigned int i = 0; 139 140 DPAA_MEMPOOL_DPDEBUG("Request to free %d buffers in bpid = %d", 141 n, bp_info->bpid); 142 143 if (unlikely(!RTE_PER_LCORE(dpaa_io))) { 144 ret = rte_dpaa_portal_init((void *)0); 145 if (ret) { 146 DPAA_MEMPOOL_ERR("rte_dpaa_portal_init failed with ret: %d", 147 ret); 148 return 0; 149 } 150 } 151 152 while (i < n) { 153 uint64_t phy = rte_mempool_virt2iova(obj_table[i]); 154 155 if (unlikely(!bp_info->ptov_off)) { 156 /* buffers are from single mem segment */ 157 if (bp_info->flags & DPAA_MPOOL_SINGLE_SEGMENT) { 158 bp_info->ptov_off = (size_t)obj_table[i] - phy; 159 rte_dpaa_bpid_info[bp_info->bpid].ptov_off 160 = bp_info->ptov_off; 161 } 162 } 163 164 dpaa_buf_free(bp_info, 165 (uint64_t)phy + bp_info->meta_data_size); 166 i = i + 1; 167 } 168 169 DPAA_MEMPOOL_DPDEBUG("freed %d buffers in bpid =%d", 170 n, bp_info->bpid); 171 172 return 0; 173 } 174 175 static int 176 dpaa_mbuf_alloc_bulk(struct rte_mempool *pool, 177 void **obj_table, 178 unsigned int count) 179 { 180 struct rte_mbuf **m = (struct rte_mbuf **)obj_table; 181 struct bm_buffer bufs[DPAA_MBUF_MAX_ACQ_REL]; 182 struct dpaa_bp_info *bp_info; 183 void *bufaddr; 184 int i, ret; 185 unsigned int n = 0; 186 187 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(pool); 188 189 DPAA_MEMPOOL_DPDEBUG("Request to alloc %d buffers in bpid = %d", 190 count, bp_info->bpid); 191 192 if (unlikely(count >= (RTE_MEMPOOL_CACHE_MAX_SIZE * 2))) { 193 DPAA_MEMPOOL_ERR("Unable to allocate requested (%u) buffers", 194 count); 195 return -1; 196 } 197 198 if (unlikely(!RTE_PER_LCORE(dpaa_io))) { 199 ret = rte_dpaa_portal_init((void *)0); 200 if (ret) { 201 DPAA_MEMPOOL_ERR("rte_dpaa_portal_init failed with ret: %d", 202 ret); 203 return -1; 204 } 205 } 206 207 while (n < count) { 208 /* Acquire is all-or-nothing, so we drain in 7s, 209 * then the remainder. 210 */ 211 if ((count - n) > DPAA_MBUF_MAX_ACQ_REL) { 212 ret = bman_acquire(bp_info->bp, bufs, 213 DPAA_MBUF_MAX_ACQ_REL, 0); 214 } else { 215 ret = bman_acquire(bp_info->bp, bufs, count - n, 0); 216 } 217 /* In case of less than requested number of buffers available 218 * in pool, qbman_swp_acquire returns 0 219 */ 220 if (ret <= 0) { 221 DPAA_MEMPOOL_DPDEBUG("Buffer acquire failed (%d)", 222 ret); 223 /* The API expect the exact number of requested 224 * buffers. Releasing all buffers allocated 225 */ 226 dpaa_mbuf_free_bulk(pool, obj_table, n); 227 return -ENOBUFS; 228 } 229 /* assigning mbuf from the acquired objects */ 230 for (i = 0; (i < ret) && bufs[i].addr; i++) { 231 /* TODO-errata - objerved that bufs may be null 232 * i.e. first buffer is valid, remaining 6 buffers 233 * may be null. 234 */ 235 bufaddr = DPAA_MEMPOOL_PTOV(bp_info, bufs[i].addr); 236 m[n] = (struct rte_mbuf *)((char *)bufaddr 237 - bp_info->meta_data_size); 238 DPAA_MEMPOOL_DPDEBUG("Paddr (%p), FD (%p) from BMAN", 239 (void *)bufaddr, (void *)m[n]); 240 n++; 241 } 242 } 243 244 DPAA_MEMPOOL_DPDEBUG("Allocated %d buffers from bpid=%d", 245 n, bp_info->bpid); 246 return 0; 247 } 248 249 static unsigned int 250 dpaa_mbuf_get_count(const struct rte_mempool *mp) 251 { 252 struct dpaa_bp_info *bp_info; 253 254 MEMPOOL_INIT_FUNC_TRACE(); 255 256 if (!mp || !mp->pool_data) { 257 DPAA_MEMPOOL_ERR("Invalid mempool provided\n"); 258 return 0; 259 } 260 261 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 262 263 return bman_query_free_buffers(bp_info->bp); 264 } 265 266 static int 267 dpaa_register_memory_area(const struct rte_mempool *mp, 268 char *vaddr __rte_unused, 269 rte_iova_t paddr __rte_unused, 270 size_t len) 271 { 272 struct dpaa_bp_info *bp_info; 273 unsigned int total_elt_sz; 274 275 MEMPOOL_INIT_FUNC_TRACE(); 276 277 if (!mp || !mp->pool_data) { 278 DPAA_MEMPOOL_ERR("Invalid mempool provided\n"); 279 return 0; 280 } 281 282 bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); 283 total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; 284 285 DPAA_MEMPOOL_DEBUG("Req size %" PRIx64 " vs Available %u\n", 286 (uint64_t)len, total_elt_sz * mp->size); 287 288 /* Detect pool area has sufficient space for elements in this memzone */ 289 if (len >= total_elt_sz * mp->size) 290 bp_info->flags |= DPAA_MPOOL_SINGLE_SEGMENT; 291 292 return 0; 293 } 294 295 struct rte_mempool_ops dpaa_mpool_ops = { 296 .name = DPAA_MEMPOOL_OPS_NAME, 297 .alloc = dpaa_mbuf_create_pool, 298 .free = dpaa_mbuf_free_pool, 299 .enqueue = dpaa_mbuf_free_bulk, 300 .dequeue = dpaa_mbuf_alloc_bulk, 301 .get_count = dpaa_mbuf_get_count, 302 .register_memory_area = dpaa_register_memory_area, 303 }; 304 305 MEMPOOL_REGISTER_OPS(dpaa_mpool_ops); 306