1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2016-2019 NXP 5 * 6 */ 7 8 #include <unistd.h> 9 #include <stdio.h> 10 #include <sys/types.h> 11 #include <string.h> 12 #include <stdlib.h> 13 #include <fcntl.h> 14 #include <errno.h> 15 16 #include <rte_mbuf.h> 17 #include <rte_ethdev_driver.h> 18 #include <rte_malloc.h> 19 #include <rte_memcpy.h> 20 #include <rte_string_fns.h> 21 #include <rte_cycles.h> 22 #include <rte_kvargs.h> 23 #include <rte_dev.h> 24 #include "rte_dpaa2_mempool.h" 25 26 #include "fslmc_vfio.h" 27 #include <fslmc_logs.h> 28 #include <mc/fsl_dpbp.h> 29 #include <portal/dpaa2_hw_pvt.h> 30 #include <portal/dpaa2_hw_dpio.h> 31 #include "dpaa2_hw_mempool.h" 32 #include "dpaa2_hw_mempool_logs.h" 33 34 #include <dpaax_iova_table.h> 35 36 struct dpaa2_bp_info *rte_dpaa2_bpid_info; 37 static struct dpaa2_bp_list *h_bp_list; 38 39 /* Dynamic logging identified for mempool */ 40 int dpaa2_logtype_mempool; 41 42 static int 43 rte_hw_mbuf_create_pool(struct rte_mempool *mp) 44 { 45 struct dpaa2_bp_list *bp_list; 46 struct dpaa2_dpbp_dev *avail_dpbp; 47 struct dpaa2_bp_info *bp_info; 48 struct dpbp_attr dpbp_attr; 49 uint32_t bpid; 50 int ret; 51 52 avail_dpbp = dpaa2_alloc_dpbp_dev(); 53 54 if (rte_dpaa2_bpid_info == NULL) { 55 rte_dpaa2_bpid_info = (struct dpaa2_bp_info *)rte_malloc(NULL, 56 sizeof(struct dpaa2_bp_info) * MAX_BPID, 57 RTE_CACHE_LINE_SIZE); 58 if (rte_dpaa2_bpid_info == NULL) 59 return -ENOMEM; 60 memset(rte_dpaa2_bpid_info, 0, 61 sizeof(struct dpaa2_bp_info) * MAX_BPID); 62 } 63 64 if (!avail_dpbp) { 65 DPAA2_MEMPOOL_ERR("DPAA2 pool not available!"); 66 return -ENOENT; 67 } 68 69 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 70 ret = dpaa2_affine_qbman_swp(); 71 if (ret) { 72 DPAA2_MEMPOOL_ERR("Failure in affining portal"); 73 goto err1; 74 } 75 } 76 77 ret = dpbp_enable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token); 78 if (ret != 0) { 79 DPAA2_MEMPOOL_ERR("Resource enable failure with err code: %d", 80 ret); 81 goto err1; 82 } 83 84 ret = dpbp_get_attributes(&avail_dpbp->dpbp, CMD_PRI_LOW, 85 avail_dpbp->token, &dpbp_attr); 86 if (ret != 0) { 87 DPAA2_MEMPOOL_ERR("Resource read failure with err code: %d", 88 ret); 89 goto err2; 90 } 91 92 bp_info = rte_malloc(NULL, 93 sizeof(struct dpaa2_bp_info), 94 RTE_CACHE_LINE_SIZE); 95 if (!bp_info) { 96 DPAA2_MEMPOOL_ERR("Unable to allocate buffer pool memory"); 97 ret = -ENOMEM; 98 goto err2; 99 } 100 101 /* Allocate the bp_list which will be added into global_bp_list */ 102 bp_list = rte_malloc(NULL, sizeof(struct dpaa2_bp_list), 103 RTE_CACHE_LINE_SIZE); 104 if (!bp_list) { 105 DPAA2_MEMPOOL_ERR("Unable to allocate buffer pool memory"); 106 ret = -ENOMEM; 107 goto err3; 108 } 109 110 /* Set parameters of buffer pool list */ 111 bp_list->buf_pool.num_bufs = mp->size; 112 bp_list->buf_pool.size = mp->elt_size 113 - sizeof(struct rte_mbuf) - rte_pktmbuf_priv_size(mp); 114 bp_list->buf_pool.bpid = dpbp_attr.bpid; 115 bp_list->buf_pool.h_bpool_mem = NULL; 116 bp_list->buf_pool.dpbp_node = avail_dpbp; 117 /* Identification for our offloaded pool_data structure */ 118 bp_list->dpaa2_ops_index = mp->ops_index; 119 bp_list->next = h_bp_list; 120 bp_list->mp = mp; 121 122 bpid = dpbp_attr.bpid; 123 124 rte_dpaa2_bpid_info[bpid].meta_data_size = sizeof(struct rte_mbuf) 125 + rte_pktmbuf_priv_size(mp); 126 rte_dpaa2_bpid_info[bpid].bp_list = bp_list; 127 rte_dpaa2_bpid_info[bpid].bpid = bpid; 128 129 rte_memcpy(bp_info, (void *)&rte_dpaa2_bpid_info[bpid], 130 sizeof(struct dpaa2_bp_info)); 131 mp->pool_data = (void *)bp_info; 132 133 DPAA2_MEMPOOL_DEBUG("BP List created for bpid =%d", dpbp_attr.bpid); 134 135 h_bp_list = bp_list; 136 return 0; 137 err3: 138 rte_free(bp_info); 139 err2: 140 dpbp_disable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token); 141 err1: 142 dpaa2_free_dpbp_dev(avail_dpbp); 143 144 return ret; 145 } 146 147 static void 148 rte_hw_mbuf_free_pool(struct rte_mempool *mp) 149 { 150 struct dpaa2_bp_info *bpinfo; 151 struct dpaa2_bp_list *bp; 152 struct dpaa2_dpbp_dev *dpbp_node; 153 154 if (!mp->pool_data) { 155 DPAA2_MEMPOOL_ERR("Not a valid dpaa2 buffer pool"); 156 return; 157 } 158 159 bpinfo = (struct dpaa2_bp_info *)mp->pool_data; 160 bp = bpinfo->bp_list; 161 dpbp_node = bp->buf_pool.dpbp_node; 162 163 dpbp_disable(&(dpbp_node->dpbp), CMD_PRI_LOW, dpbp_node->token); 164 165 if (h_bp_list == bp) { 166 h_bp_list = h_bp_list->next; 167 } else { /* if it is not the first node */ 168 struct dpaa2_bp_list *prev = h_bp_list, *temp; 169 temp = h_bp_list->next; 170 while (temp) { 171 if (temp == bp) { 172 prev->next = temp->next; 173 rte_free(bp); 174 break; 175 } 176 prev = temp; 177 temp = temp->next; 178 } 179 } 180 181 rte_free(mp->pool_data); 182 dpaa2_free_dpbp_dev(dpbp_node); 183 } 184 185 static void 186 rte_dpaa2_mbuf_release(struct rte_mempool *pool __rte_unused, 187 void * const *obj_table, 188 uint32_t bpid, 189 uint32_t meta_data_size, 190 int count) 191 { 192 struct qbman_release_desc releasedesc; 193 struct qbman_swp *swp; 194 int ret; 195 int i, n, retry_count; 196 uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL]; 197 198 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 199 ret = dpaa2_affine_qbman_swp(); 200 if (ret != 0) { 201 DPAA2_MEMPOOL_ERR("Failed to allocate IO portal"); 202 return; 203 } 204 } 205 swp = DPAA2_PER_LCORE_PORTAL; 206 207 /* Create a release descriptor required for releasing 208 * buffers into QBMAN 209 */ 210 qbman_release_desc_clear(&releasedesc); 211 qbman_release_desc_set_bpid(&releasedesc, bpid); 212 213 n = count % DPAA2_MBUF_MAX_ACQ_REL; 214 if (unlikely(!n)) 215 goto aligned; 216 217 /* convert mbuf to buffers for the remainder */ 218 for (i = 0; i < n ; i++) { 219 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA 220 bufs[i] = (uint64_t)rte_mempool_virt2iova(obj_table[i]) 221 + meta_data_size; 222 #else 223 bufs[i] = (uint64_t)obj_table[i] + meta_data_size; 224 #endif 225 } 226 227 /* feed them to bman */ 228 retry_count = 0; 229 while ((ret = qbman_swp_release(swp, &releasedesc, bufs, n)) == 230 -EBUSY) { 231 retry_count++; 232 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) { 233 DPAA2_MEMPOOL_ERR("bman release retry exceeded, low fbpr?"); 234 return; 235 } 236 } 237 238 aligned: 239 /* if there are more buffers to free */ 240 while (n < count) { 241 /* convert mbuf to buffers */ 242 for (i = 0; i < DPAA2_MBUF_MAX_ACQ_REL; i++) { 243 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA 244 bufs[i] = (uint64_t) 245 rte_mempool_virt2iova(obj_table[n + i]) 246 + meta_data_size; 247 #else 248 bufs[i] = (uint64_t)obj_table[n + i] + meta_data_size; 249 #endif 250 } 251 252 retry_count = 0; 253 while ((ret = qbman_swp_release(swp, &releasedesc, bufs, 254 DPAA2_MBUF_MAX_ACQ_REL)) == -EBUSY) { 255 retry_count++; 256 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) { 257 DPAA2_MEMPOOL_ERR("bman release retry exceeded, low fbpr?"); 258 return; 259 } 260 } 261 n += DPAA2_MBUF_MAX_ACQ_REL; 262 } 263 } 264 265 uint16_t 266 rte_dpaa2_mbuf_pool_bpid(struct rte_mempool *mp) 267 { 268 struct dpaa2_bp_info *bp_info; 269 270 bp_info = mempool_to_bpinfo(mp); 271 if (!(bp_info->bp_list)) { 272 RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n"); 273 return -ENOMEM; 274 } 275 276 return bp_info->bpid; 277 } 278 279 struct rte_mbuf * 280 rte_dpaa2_mbuf_from_buf_addr(struct rte_mempool *mp, void *buf_addr) 281 { 282 struct dpaa2_bp_info *bp_info; 283 284 bp_info = mempool_to_bpinfo(mp); 285 if (!(bp_info->bp_list)) { 286 RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n"); 287 return NULL; 288 } 289 290 return (struct rte_mbuf *)((uint8_t *)buf_addr - 291 bp_info->meta_data_size); 292 } 293 294 int 295 rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool, 296 void **obj_table, unsigned int count) 297 { 298 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER 299 static int alloc; 300 #endif 301 struct qbman_swp *swp; 302 uint16_t bpid; 303 size_t bufs[DPAA2_MBUF_MAX_ACQ_REL]; 304 int i, ret; 305 unsigned int n = 0; 306 struct dpaa2_bp_info *bp_info; 307 308 bp_info = mempool_to_bpinfo(pool); 309 310 if (!(bp_info->bp_list)) { 311 DPAA2_MEMPOOL_ERR("DPAA2 buffer pool not configured"); 312 return -ENOENT; 313 } 314 315 bpid = bp_info->bpid; 316 317 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 318 ret = dpaa2_affine_qbman_swp(); 319 if (ret != 0) { 320 DPAA2_MEMPOOL_ERR("Failed to allocate IO portal"); 321 return ret; 322 } 323 } 324 swp = DPAA2_PER_LCORE_PORTAL; 325 326 while (n < count) { 327 /* Acquire is all-or-nothing, so we drain in 7s, 328 * then the remainder. 329 */ 330 if ((count - n) > DPAA2_MBUF_MAX_ACQ_REL) { 331 ret = qbman_swp_acquire(swp, bpid, (void *)bufs, 332 DPAA2_MBUF_MAX_ACQ_REL); 333 } else { 334 ret = qbman_swp_acquire(swp, bpid, (void *)bufs, 335 count - n); 336 } 337 /* In case of less than requested number of buffers available 338 * in pool, qbman_swp_acquire returns 0 339 */ 340 if (ret <= 0) { 341 DPAA2_MEMPOOL_DP_DEBUG( 342 "Buffer acquire failed with err code: %d", ret); 343 /* The API expect the exact number of requested bufs */ 344 /* Releasing all buffers allocated */ 345 rte_dpaa2_mbuf_release(pool, obj_table, bpid, 346 bp_info->meta_data_size, n); 347 return -ENOBUFS; 348 } 349 /* assigning mbuf from the acquired objects */ 350 for (i = 0; (i < ret) && bufs[i]; i++) { 351 DPAA2_MODIFY_IOVA_TO_VADDR(bufs[i], size_t); 352 obj_table[n] = (struct rte_mbuf *) 353 (bufs[i] - bp_info->meta_data_size); 354 DPAA2_MEMPOOL_DP_DEBUG( 355 "Acquired %p address %p from BMAN\n", 356 (void *)bufs[i], (void *)obj_table[n]); 357 n++; 358 } 359 } 360 361 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER 362 alloc += n; 363 DPAA2_MEMPOOL_DP_DEBUG("Total = %d , req = %d done = %d\n", 364 alloc, count, n); 365 #endif 366 return 0; 367 } 368 369 static int 370 rte_hw_mbuf_free_bulk(struct rte_mempool *pool, 371 void * const *obj_table, unsigned int n) 372 { 373 struct dpaa2_bp_info *bp_info; 374 375 bp_info = mempool_to_bpinfo(pool); 376 if (!(bp_info->bp_list)) { 377 DPAA2_MEMPOOL_ERR("DPAA2 buffer pool not configured"); 378 return -ENOENT; 379 } 380 rte_dpaa2_mbuf_release(pool, obj_table, bp_info->bpid, 381 bp_info->meta_data_size, n); 382 383 return 0; 384 } 385 386 static unsigned int 387 rte_hw_mbuf_get_count(const struct rte_mempool *mp) 388 { 389 int ret; 390 unsigned int num_of_bufs = 0; 391 struct dpaa2_bp_info *bp_info; 392 struct dpaa2_dpbp_dev *dpbp_node; 393 394 if (!mp || !mp->pool_data) { 395 DPAA2_MEMPOOL_ERR("Invalid mempool provided"); 396 return 0; 397 } 398 399 bp_info = (struct dpaa2_bp_info *)mp->pool_data; 400 dpbp_node = bp_info->bp_list->buf_pool.dpbp_node; 401 402 ret = dpbp_get_num_free_bufs(&dpbp_node->dpbp, CMD_PRI_LOW, 403 dpbp_node->token, &num_of_bufs); 404 if (ret) { 405 DPAA2_MEMPOOL_ERR("Unable to obtain free buf count (err=%d)", 406 ret); 407 return 0; 408 } 409 410 DPAA2_MEMPOOL_DP_DEBUG("Free bufs = %u\n", num_of_bufs); 411 412 return num_of_bufs; 413 } 414 415 static int 416 dpaa2_populate(struct rte_mempool *mp, unsigned int max_objs, 417 void *vaddr, rte_iova_t paddr, size_t len, 418 rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg) 419 { 420 struct rte_memseg_list *msl; 421 /* The memsegment list exists incase the memory is not external. 422 * So, DMA-Map is required only when memory is provided by user, 423 * i.e. External. 424 */ 425 msl = rte_mem_virt2memseg_list(vaddr); 426 427 if (!msl) { 428 DPAA2_MEMPOOL_DEBUG("Memsegment is External.\n"); 429 rte_fslmc_vfio_mem_dmamap((size_t)vaddr, 430 (size_t)paddr, (size_t)len); 431 } 432 /* Insert entry into the PA->VA Table */ 433 dpaax_iova_table_update(paddr, vaddr, len); 434 435 return rte_mempool_op_populate_helper(mp, 0, max_objs, vaddr, paddr, 436 len, obj_cb, obj_cb_arg); 437 } 438 439 static const struct rte_mempool_ops dpaa2_mpool_ops = { 440 .name = DPAA2_MEMPOOL_OPS_NAME, 441 .alloc = rte_hw_mbuf_create_pool, 442 .free = rte_hw_mbuf_free_pool, 443 .enqueue = rte_hw_mbuf_free_bulk, 444 .dequeue = rte_dpaa2_mbuf_alloc_bulk, 445 .get_count = rte_hw_mbuf_get_count, 446 .populate = dpaa2_populate, 447 }; 448 449 MEMPOOL_REGISTER_OPS(dpaa2_mpool_ops); 450 451 RTE_INIT(dpaa2_mempool_init_log) 452 { 453 dpaa2_logtype_mempool = rte_log_register("mempool.dpaa2"); 454 if (dpaa2_logtype_mempool >= 0) 455 rte_log_set_level(dpaa2_logtype_mempool, RTE_LOG_NOTICE); 456 } 457