1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2016 NXP 5 * 6 */ 7 8 #include <unistd.h> 9 #include <stdio.h> 10 #include <sys/types.h> 11 #include <string.h> 12 #include <stdlib.h> 13 #include <fcntl.h> 14 #include <errno.h> 15 16 #include <rte_mbuf.h> 17 #include <rte_ethdev_driver.h> 18 #include <rte_malloc.h> 19 #include <rte_memcpy.h> 20 #include <rte_string_fns.h> 21 #include <rte_cycles.h> 22 #include <rte_kvargs.h> 23 #include <rte_dev.h> 24 #include "rte_dpaa2_mempool.h" 25 26 #include "fslmc_vfio.h" 27 #include <fslmc_logs.h> 28 #include <mc/fsl_dpbp.h> 29 #include <portal/dpaa2_hw_pvt.h> 30 #include <portal/dpaa2_hw_dpio.h> 31 #include "dpaa2_hw_mempool.h" 32 #include "dpaa2_hw_mempool_logs.h" 33 34 #include <dpaax_iova_table.h> 35 36 struct dpaa2_bp_info *rte_dpaa2_bpid_info; 37 static struct dpaa2_bp_list *h_bp_list; 38 39 /* Dynamic logging identified for mempool */ 40 int dpaa2_logtype_mempool; 41 42 static int 43 rte_hw_mbuf_create_pool(struct rte_mempool *mp) 44 { 45 struct dpaa2_bp_list *bp_list; 46 struct dpaa2_dpbp_dev *avail_dpbp; 47 struct dpaa2_bp_info *bp_info; 48 struct dpbp_attr dpbp_attr; 49 uint32_t bpid; 50 int ret; 51 52 avail_dpbp = dpaa2_alloc_dpbp_dev(); 53 54 if (rte_dpaa2_bpid_info == NULL) { 55 rte_dpaa2_bpid_info = (struct dpaa2_bp_info *)rte_malloc(NULL, 56 sizeof(struct dpaa2_bp_info) * MAX_BPID, 57 RTE_CACHE_LINE_SIZE); 58 if (rte_dpaa2_bpid_info == NULL) 59 return -ENOMEM; 60 memset(rte_dpaa2_bpid_info, 0, 61 sizeof(struct dpaa2_bp_info) * MAX_BPID); 62 } 63 64 if (!avail_dpbp) { 65 DPAA2_MEMPOOL_ERR("DPAA2 pool not available!"); 66 return -ENOENT; 67 } 68 69 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 70 ret = dpaa2_affine_qbman_swp(); 71 if (ret) { 72 DPAA2_MEMPOOL_ERR("Failure in affining portal"); 73 goto err1; 74 } 75 } 76 77 ret = dpbp_enable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token); 78 if (ret != 0) { 79 DPAA2_MEMPOOL_ERR("Resource enable failure with err code: %d", 80 ret); 81 goto err1; 82 } 83 84 ret = dpbp_get_attributes(&avail_dpbp->dpbp, CMD_PRI_LOW, 85 avail_dpbp->token, &dpbp_attr); 86 if (ret != 0) { 87 DPAA2_MEMPOOL_ERR("Resource read failure with err code: %d", 88 ret); 89 goto err2; 90 } 91 92 bp_info = rte_malloc(NULL, 93 sizeof(struct dpaa2_bp_info), 94 RTE_CACHE_LINE_SIZE); 95 if (!bp_info) { 96 DPAA2_MEMPOOL_ERR("Unable to allocate buffer pool memory"); 97 ret = -ENOMEM; 98 goto err2; 99 } 100 101 /* Allocate the bp_list which will be added into global_bp_list */ 102 bp_list = rte_malloc(NULL, sizeof(struct dpaa2_bp_list), 103 RTE_CACHE_LINE_SIZE); 104 if (!bp_list) { 105 DPAA2_MEMPOOL_ERR("Unable to allocate buffer pool memory"); 106 ret = -ENOMEM; 107 goto err3; 108 } 109 110 /* Set parameters of buffer pool list */ 111 bp_list->buf_pool.num_bufs = mp->size; 112 bp_list->buf_pool.size = mp->elt_size 113 - sizeof(struct rte_mbuf) - rte_pktmbuf_priv_size(mp); 114 bp_list->buf_pool.bpid = dpbp_attr.bpid; 115 bp_list->buf_pool.h_bpool_mem = NULL; 116 bp_list->buf_pool.dpbp_node = avail_dpbp; 117 /* Identification for our offloaded pool_data structure */ 118 bp_list->dpaa2_ops_index = mp->ops_index; 119 bp_list->next = h_bp_list; 120 bp_list->mp = mp; 121 122 bpid = dpbp_attr.bpid; 123 124 rte_dpaa2_bpid_info[bpid].meta_data_size = sizeof(struct rte_mbuf) 125 + rte_pktmbuf_priv_size(mp); 126 rte_dpaa2_bpid_info[bpid].bp_list = bp_list; 127 rte_dpaa2_bpid_info[bpid].bpid = bpid; 128 129 rte_memcpy(bp_info, (void *)&rte_dpaa2_bpid_info[bpid], 130 sizeof(struct dpaa2_bp_info)); 131 mp->pool_data = (void *)bp_info; 132 133 DPAA2_MEMPOOL_DEBUG("BP List created for bpid =%d", dpbp_attr.bpid); 134 135 h_bp_list = bp_list; 136 return 0; 137 err3: 138 rte_free(bp_info); 139 err2: 140 dpbp_disable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token); 141 err1: 142 dpaa2_free_dpbp_dev(avail_dpbp); 143 144 return ret; 145 } 146 147 static void 148 rte_hw_mbuf_free_pool(struct rte_mempool *mp) 149 { 150 struct dpaa2_bp_info *bpinfo; 151 struct dpaa2_bp_list *bp; 152 struct dpaa2_dpbp_dev *dpbp_node; 153 154 if (!mp->pool_data) { 155 DPAA2_MEMPOOL_ERR("Not a valid dpaa2 buffer pool"); 156 return; 157 } 158 159 bpinfo = (struct dpaa2_bp_info *)mp->pool_data; 160 bp = bpinfo->bp_list; 161 dpbp_node = bp->buf_pool.dpbp_node; 162 163 dpbp_disable(&(dpbp_node->dpbp), CMD_PRI_LOW, dpbp_node->token); 164 165 if (h_bp_list == bp) { 166 h_bp_list = h_bp_list->next; 167 } else { /* if it is not the first node */ 168 struct dpaa2_bp_list *prev = h_bp_list, *temp; 169 temp = h_bp_list->next; 170 while (temp) { 171 if (temp == bp) { 172 prev->next = temp->next; 173 rte_free(bp); 174 break; 175 } 176 prev = temp; 177 temp = temp->next; 178 } 179 } 180 181 rte_free(mp->pool_data); 182 dpaa2_free_dpbp_dev(dpbp_node); 183 } 184 185 static void 186 rte_dpaa2_mbuf_release(struct rte_mempool *pool __rte_unused, 187 void * const *obj_table, 188 uint32_t bpid, 189 uint32_t meta_data_size, 190 int count) 191 { 192 struct qbman_release_desc releasedesc; 193 struct qbman_swp *swp; 194 int ret; 195 int i, n; 196 uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL]; 197 198 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 199 ret = dpaa2_affine_qbman_swp(); 200 if (ret != 0) { 201 DPAA2_MEMPOOL_ERR("Failed to allocate IO portal"); 202 return; 203 } 204 } 205 swp = DPAA2_PER_LCORE_PORTAL; 206 207 /* Create a release descriptor required for releasing 208 * buffers into QBMAN 209 */ 210 qbman_release_desc_clear(&releasedesc); 211 qbman_release_desc_set_bpid(&releasedesc, bpid); 212 213 n = count % DPAA2_MBUF_MAX_ACQ_REL; 214 if (unlikely(!n)) 215 goto aligned; 216 217 /* convert mbuf to buffers for the remainder */ 218 for (i = 0; i < n ; i++) { 219 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA 220 bufs[i] = (uint64_t)rte_mempool_virt2iova(obj_table[i]) 221 + meta_data_size; 222 #else 223 bufs[i] = (uint64_t)obj_table[i] + meta_data_size; 224 #endif 225 } 226 227 /* feed them to bman */ 228 do { 229 ret = qbman_swp_release(swp, &releasedesc, bufs, n); 230 } while (ret == -EBUSY); 231 232 aligned: 233 /* if there are more buffers to free */ 234 while (n < count) { 235 /* convert mbuf to buffers */ 236 for (i = 0; i < DPAA2_MBUF_MAX_ACQ_REL; i++) { 237 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA 238 bufs[i] = (uint64_t) 239 rte_mempool_virt2iova(obj_table[n + i]) 240 + meta_data_size; 241 #else 242 bufs[i] = (uint64_t)obj_table[n + i] + meta_data_size; 243 #endif 244 } 245 246 do { 247 ret = qbman_swp_release(swp, &releasedesc, bufs, 248 DPAA2_MBUF_MAX_ACQ_REL); 249 } while (ret == -EBUSY); 250 n += DPAA2_MBUF_MAX_ACQ_REL; 251 } 252 } 253 254 uint16_t 255 rte_dpaa2_mbuf_pool_bpid(struct rte_mempool *mp) 256 { 257 struct dpaa2_bp_info *bp_info; 258 259 bp_info = mempool_to_bpinfo(mp); 260 if (!(bp_info->bp_list)) { 261 RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n"); 262 return -ENOMEM; 263 } 264 265 return bp_info->bpid; 266 } 267 268 struct rte_mbuf * 269 rte_dpaa2_mbuf_from_buf_addr(struct rte_mempool *mp, void *buf_addr) 270 { 271 struct dpaa2_bp_info *bp_info; 272 273 bp_info = mempool_to_bpinfo(mp); 274 if (!(bp_info->bp_list)) { 275 RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n"); 276 return NULL; 277 } 278 279 return (struct rte_mbuf *)((uint8_t *)buf_addr - 280 bp_info->meta_data_size); 281 } 282 283 int 284 rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool, 285 void **obj_table, unsigned int count) 286 { 287 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER 288 static int alloc; 289 #endif 290 struct qbman_swp *swp; 291 uint16_t bpid; 292 size_t bufs[DPAA2_MBUF_MAX_ACQ_REL]; 293 int i, ret; 294 unsigned int n = 0; 295 struct dpaa2_bp_info *bp_info; 296 297 bp_info = mempool_to_bpinfo(pool); 298 299 if (!(bp_info->bp_list)) { 300 DPAA2_MEMPOOL_ERR("DPAA2 buffer pool not configured"); 301 return -ENOENT; 302 } 303 304 bpid = bp_info->bpid; 305 306 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 307 ret = dpaa2_affine_qbman_swp(); 308 if (ret != 0) { 309 DPAA2_MEMPOOL_ERR("Failed to allocate IO portal"); 310 return ret; 311 } 312 } 313 swp = DPAA2_PER_LCORE_PORTAL; 314 315 while (n < count) { 316 /* Acquire is all-or-nothing, so we drain in 7s, 317 * then the remainder. 318 */ 319 if ((count - n) > DPAA2_MBUF_MAX_ACQ_REL) { 320 ret = qbman_swp_acquire(swp, bpid, (void *)bufs, 321 DPAA2_MBUF_MAX_ACQ_REL); 322 } else { 323 ret = qbman_swp_acquire(swp, bpid, (void *)bufs, 324 count - n); 325 } 326 /* In case of less than requested number of buffers available 327 * in pool, qbman_swp_acquire returns 0 328 */ 329 if (ret <= 0) { 330 DPAA2_MEMPOOL_DP_DEBUG( 331 "Buffer acquire failed with err code: %d", ret); 332 /* The API expect the exact number of requested bufs */ 333 /* Releasing all buffers allocated */ 334 rte_dpaa2_mbuf_release(pool, obj_table, bpid, 335 bp_info->meta_data_size, n); 336 return -ENOBUFS; 337 } 338 /* assigning mbuf from the acquired objects */ 339 for (i = 0; (i < ret) && bufs[i]; i++) { 340 DPAA2_MODIFY_IOVA_TO_VADDR(bufs[i], size_t); 341 obj_table[n] = (struct rte_mbuf *) 342 (bufs[i] - bp_info->meta_data_size); 343 DPAA2_MEMPOOL_DP_DEBUG( 344 "Acquired %p address %p from BMAN\n", 345 (void *)bufs[i], (void *)obj_table[n]); 346 n++; 347 } 348 } 349 350 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER 351 alloc += n; 352 DPAA2_MEMPOOL_DP_DEBUG("Total = %d , req = %d done = %d\n", 353 alloc, count, n); 354 #endif 355 return 0; 356 } 357 358 static int 359 rte_hw_mbuf_free_bulk(struct rte_mempool *pool, 360 void * const *obj_table, unsigned int n) 361 { 362 struct dpaa2_bp_info *bp_info; 363 364 bp_info = mempool_to_bpinfo(pool); 365 if (!(bp_info->bp_list)) { 366 DPAA2_MEMPOOL_ERR("DPAA2 buffer pool not configured"); 367 return -ENOENT; 368 } 369 rte_dpaa2_mbuf_release(pool, obj_table, bp_info->bpid, 370 bp_info->meta_data_size, n); 371 372 return 0; 373 } 374 375 static unsigned int 376 rte_hw_mbuf_get_count(const struct rte_mempool *mp) 377 { 378 int ret; 379 unsigned int num_of_bufs = 0; 380 struct dpaa2_bp_info *bp_info; 381 struct dpaa2_dpbp_dev *dpbp_node; 382 383 if (!mp || !mp->pool_data) { 384 DPAA2_MEMPOOL_ERR("Invalid mempool provided"); 385 return 0; 386 } 387 388 bp_info = (struct dpaa2_bp_info *)mp->pool_data; 389 dpbp_node = bp_info->bp_list->buf_pool.dpbp_node; 390 391 ret = dpbp_get_num_free_bufs(&dpbp_node->dpbp, CMD_PRI_LOW, 392 dpbp_node->token, &num_of_bufs); 393 if (ret) { 394 DPAA2_MEMPOOL_ERR("Unable to obtain free buf count (err=%d)", 395 ret); 396 return 0; 397 } 398 399 DPAA2_MEMPOOL_DP_DEBUG("Free bufs = %u\n", num_of_bufs); 400 401 return num_of_bufs; 402 } 403 404 static int 405 dpaa2_populate(struct rte_mempool *mp, unsigned int max_objs, 406 void *vaddr, rte_iova_t paddr, size_t len, 407 rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg) 408 { 409 struct rte_memseg_list *msl; 410 /* The memsegment list exists incase the memory is not external. 411 * So, DMA-Map is required only when memory is provided by user, 412 * i.e. External. 413 */ 414 msl = rte_mem_virt2memseg_list(vaddr); 415 416 if (!msl) { 417 DPAA2_MEMPOOL_DEBUG("Memsegment is External.\n"); 418 rte_fslmc_vfio_mem_dmamap((size_t)vaddr, 419 (size_t)paddr, (size_t)len); 420 } 421 /* Insert entry into the PA->VA Table */ 422 dpaax_iova_table_update(paddr, vaddr, len); 423 424 return rte_mempool_op_populate_default(mp, max_objs, vaddr, paddr, len, 425 obj_cb, obj_cb_arg); 426 } 427 428 static const struct rte_mempool_ops dpaa2_mpool_ops = { 429 .name = DPAA2_MEMPOOL_OPS_NAME, 430 .alloc = rte_hw_mbuf_create_pool, 431 .free = rte_hw_mbuf_free_pool, 432 .enqueue = rte_hw_mbuf_free_bulk, 433 .dequeue = rte_dpaa2_mbuf_alloc_bulk, 434 .get_count = rte_hw_mbuf_get_count, 435 .populate = dpaa2_populate, 436 }; 437 438 MEMPOOL_REGISTER_OPS(dpaa2_mpool_ops); 439 440 RTE_INIT(dpaa2_mempool_init_log) 441 { 442 dpaa2_logtype_mempool = rte_log_register("mempool.dpaa2"); 443 if (dpaa2_logtype_mempool >= 0) 444 rte_log_set_level(dpaa2_logtype_mempool, RTE_LOG_NOTICE); 445 } 446