1 /* SPDX-License-Identifier: BSD-3-Clause
2  *
3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
4  *   Copyright 2016 NXP
5  *
6  */
7 
8 #include <unistd.h>
9 #include <stdio.h>
10 #include <sys/types.h>
11 #include <string.h>
12 #include <stdlib.h>
13 #include <fcntl.h>
14 #include <errno.h>
15 
16 #include <rte_mbuf.h>
17 #include <rte_ethdev_driver.h>
18 #include <rte_malloc.h>
19 #include <rte_memcpy.h>
20 #include <rte_string_fns.h>
21 #include <rte_cycles.h>
22 #include <rte_kvargs.h>
23 #include <rte_dev.h>
24 #include "rte_dpaa2_mempool.h"
25 
26 #include <fslmc_logs.h>
27 #include <mc/fsl_dpbp.h>
28 #include <portal/dpaa2_hw_pvt.h>
29 #include <portal/dpaa2_hw_dpio.h>
30 #include "dpaa2_hw_mempool.h"
31 #include "dpaa2_hw_mempool_logs.h"
32 
33 #include <dpaax_iova_table.h>
34 
35 struct dpaa2_bp_info *rte_dpaa2_bpid_info;
36 static struct dpaa2_bp_list *h_bp_list;
37 
38 /* Dynamic logging identified for mempool */
39 int dpaa2_logtype_mempool;
40 
41 static int
42 rte_hw_mbuf_create_pool(struct rte_mempool *mp)
43 {
44 	struct dpaa2_bp_list *bp_list;
45 	struct dpaa2_dpbp_dev *avail_dpbp;
46 	struct dpaa2_bp_info *bp_info;
47 	struct dpbp_attr dpbp_attr;
48 	uint32_t bpid;
49 	int ret;
50 
51 	avail_dpbp = dpaa2_alloc_dpbp_dev();
52 
53 	if (rte_dpaa2_bpid_info == NULL) {
54 		rte_dpaa2_bpid_info = (struct dpaa2_bp_info *)rte_malloc(NULL,
55 				      sizeof(struct dpaa2_bp_info) * MAX_BPID,
56 				      RTE_CACHE_LINE_SIZE);
57 		if (rte_dpaa2_bpid_info == NULL)
58 			return -ENOMEM;
59 		memset(rte_dpaa2_bpid_info, 0,
60 		       sizeof(struct dpaa2_bp_info) * MAX_BPID);
61 	}
62 
63 	if (!avail_dpbp) {
64 		DPAA2_MEMPOOL_ERR("DPAA2 pool not available!");
65 		return -ENOENT;
66 	}
67 
68 	if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
69 		ret = dpaa2_affine_qbman_swp();
70 		if (ret) {
71 			DPAA2_MEMPOOL_ERR("Failure in affining portal");
72 			goto err1;
73 		}
74 	}
75 
76 	ret = dpbp_enable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token);
77 	if (ret != 0) {
78 		DPAA2_MEMPOOL_ERR("Resource enable failure with err code: %d",
79 				  ret);
80 		goto err1;
81 	}
82 
83 	ret = dpbp_get_attributes(&avail_dpbp->dpbp, CMD_PRI_LOW,
84 				  avail_dpbp->token, &dpbp_attr);
85 	if (ret != 0) {
86 		DPAA2_MEMPOOL_ERR("Resource read failure with err code: %d",
87 				  ret);
88 		goto err2;
89 	}
90 
91 	bp_info = rte_malloc(NULL,
92 			     sizeof(struct dpaa2_bp_info),
93 			     RTE_CACHE_LINE_SIZE);
94 	if (!bp_info) {
95 		DPAA2_MEMPOOL_ERR("Unable to allocate buffer pool memory");
96 		ret = -ENOMEM;
97 		goto err2;
98 	}
99 
100 	/* Allocate the bp_list which will be added into global_bp_list */
101 	bp_list = rte_malloc(NULL, sizeof(struct dpaa2_bp_list),
102 			     RTE_CACHE_LINE_SIZE);
103 	if (!bp_list) {
104 		DPAA2_MEMPOOL_ERR("Unable to allocate buffer pool memory");
105 		ret = -ENOMEM;
106 		goto err3;
107 	}
108 
109 	/* Set parameters of buffer pool list */
110 	bp_list->buf_pool.num_bufs = mp->size;
111 	bp_list->buf_pool.size = mp->elt_size
112 			- sizeof(struct rte_mbuf) - rte_pktmbuf_priv_size(mp);
113 	bp_list->buf_pool.bpid = dpbp_attr.bpid;
114 	bp_list->buf_pool.h_bpool_mem = NULL;
115 	bp_list->buf_pool.dpbp_node = avail_dpbp;
116 	/* Identification for our offloaded pool_data structure */
117 	bp_list->dpaa2_ops_index = mp->ops_index;
118 	bp_list->next = h_bp_list;
119 	bp_list->mp = mp;
120 
121 	bpid = dpbp_attr.bpid;
122 
123 	rte_dpaa2_bpid_info[bpid].meta_data_size = sizeof(struct rte_mbuf)
124 				+ rte_pktmbuf_priv_size(mp);
125 	rte_dpaa2_bpid_info[bpid].bp_list = bp_list;
126 	rte_dpaa2_bpid_info[bpid].bpid = bpid;
127 
128 	rte_memcpy(bp_info, (void *)&rte_dpaa2_bpid_info[bpid],
129 		   sizeof(struct dpaa2_bp_info));
130 	mp->pool_data = (void *)bp_info;
131 
132 	DPAA2_MEMPOOL_DEBUG("BP List created for bpid =%d", dpbp_attr.bpid);
133 
134 	h_bp_list = bp_list;
135 	return 0;
136 err3:
137 	rte_free(bp_info);
138 err2:
139 	dpbp_disable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token);
140 err1:
141 	dpaa2_free_dpbp_dev(avail_dpbp);
142 
143 	return ret;
144 }
145 
146 static void
147 rte_hw_mbuf_free_pool(struct rte_mempool *mp)
148 {
149 	struct dpaa2_bp_info *bpinfo;
150 	struct dpaa2_bp_list *bp;
151 	struct dpaa2_dpbp_dev *dpbp_node;
152 
153 	if (!mp->pool_data) {
154 		DPAA2_MEMPOOL_ERR("Not a valid dpaa2 buffer pool");
155 		return;
156 	}
157 
158 	bpinfo = (struct dpaa2_bp_info *)mp->pool_data;
159 	bp = bpinfo->bp_list;
160 	dpbp_node = bp->buf_pool.dpbp_node;
161 
162 	dpbp_disable(&(dpbp_node->dpbp), CMD_PRI_LOW, dpbp_node->token);
163 
164 	if (h_bp_list == bp) {
165 		h_bp_list = h_bp_list->next;
166 	} else { /* if it is not the first node */
167 		struct dpaa2_bp_list *prev = h_bp_list, *temp;
168 		temp = h_bp_list->next;
169 		while (temp) {
170 			if (temp == bp) {
171 				prev->next = temp->next;
172 				rte_free(bp);
173 				break;
174 			}
175 			prev = temp;
176 			temp = temp->next;
177 		}
178 	}
179 
180 	rte_free(mp->pool_data);
181 	dpaa2_free_dpbp_dev(dpbp_node);
182 }
183 
184 static void
185 rte_dpaa2_mbuf_release(struct rte_mempool *pool __rte_unused,
186 			void * const *obj_table,
187 			uint32_t bpid,
188 			uint32_t meta_data_size,
189 			int count)
190 {
191 	struct qbman_release_desc releasedesc;
192 	struct qbman_swp *swp;
193 	int ret;
194 	int i, n;
195 	uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL];
196 
197 	if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
198 		ret = dpaa2_affine_qbman_swp();
199 		if (ret != 0) {
200 			DPAA2_MEMPOOL_ERR("Failed to allocate IO portal");
201 			return;
202 		}
203 	}
204 	swp = DPAA2_PER_LCORE_PORTAL;
205 
206 	/* Create a release descriptor required for releasing
207 	 * buffers into QBMAN
208 	 */
209 	qbman_release_desc_clear(&releasedesc);
210 	qbman_release_desc_set_bpid(&releasedesc, bpid);
211 
212 	n = count % DPAA2_MBUF_MAX_ACQ_REL;
213 	if (unlikely(!n))
214 		goto aligned;
215 
216 	/* convert mbuf to buffers for the remainder */
217 	for (i = 0; i < n ; i++) {
218 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
219 		bufs[i] = (uint64_t)rte_mempool_virt2iova(obj_table[i])
220 				+ meta_data_size;
221 #else
222 		bufs[i] = (uint64_t)obj_table[i] + meta_data_size;
223 #endif
224 	}
225 
226 	/* feed them to bman */
227 	do {
228 		ret = qbman_swp_release(swp, &releasedesc, bufs, n);
229 	} while (ret == -EBUSY);
230 
231 aligned:
232 	/* if there are more buffers to free */
233 	while (n < count) {
234 		/* convert mbuf to buffers */
235 		for (i = 0; i < DPAA2_MBUF_MAX_ACQ_REL; i++) {
236 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
237 			bufs[i] = (uint64_t)
238 				  rte_mempool_virt2iova(obj_table[n + i])
239 				  + meta_data_size;
240 #else
241 			bufs[i] = (uint64_t)obj_table[n + i] + meta_data_size;
242 #endif
243 		}
244 
245 		do {
246 			ret = qbman_swp_release(swp, &releasedesc, bufs,
247 						DPAA2_MBUF_MAX_ACQ_REL);
248 		} while (ret == -EBUSY);
249 		n += DPAA2_MBUF_MAX_ACQ_REL;
250 	}
251 }
252 
253 uint16_t
254 rte_dpaa2_mbuf_pool_bpid(struct rte_mempool *mp)
255 {
256 	struct dpaa2_bp_info *bp_info;
257 
258 	bp_info = mempool_to_bpinfo(mp);
259 	if (!(bp_info->bp_list)) {
260 		RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n");
261 		return -ENOMEM;
262 	}
263 
264 	return bp_info->bpid;
265 }
266 
267 struct rte_mbuf *
268 rte_dpaa2_mbuf_from_buf_addr(struct rte_mempool *mp, void *buf_addr)
269 {
270 	struct dpaa2_bp_info *bp_info;
271 
272 	bp_info = mempool_to_bpinfo(mp);
273 	if (!(bp_info->bp_list)) {
274 		RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n");
275 		return NULL;
276 	}
277 
278 	return (struct rte_mbuf *)((uint8_t *)buf_addr -
279 			bp_info->meta_data_size);
280 }
281 
282 int
283 rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool,
284 			  void **obj_table, unsigned int count)
285 {
286 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER
287 	static int alloc;
288 #endif
289 	struct qbman_swp *swp;
290 	uint16_t bpid;
291 	size_t bufs[DPAA2_MBUF_MAX_ACQ_REL];
292 	int i, ret;
293 	unsigned int n = 0;
294 	struct dpaa2_bp_info *bp_info;
295 
296 	bp_info = mempool_to_bpinfo(pool);
297 
298 	if (!(bp_info->bp_list)) {
299 		DPAA2_MEMPOOL_ERR("DPAA2 buffer pool not configured");
300 		return -ENOENT;
301 	}
302 
303 	bpid = bp_info->bpid;
304 
305 	if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
306 		ret = dpaa2_affine_qbman_swp();
307 		if (ret != 0) {
308 			DPAA2_MEMPOOL_ERR("Failed to allocate IO portal");
309 			return ret;
310 		}
311 	}
312 	swp = DPAA2_PER_LCORE_PORTAL;
313 
314 	while (n < count) {
315 		/* Acquire is all-or-nothing, so we drain in 7s,
316 		 * then the remainder.
317 		 */
318 		if ((count - n) > DPAA2_MBUF_MAX_ACQ_REL) {
319 			ret = qbman_swp_acquire(swp, bpid, (void *)bufs,
320 						DPAA2_MBUF_MAX_ACQ_REL);
321 		} else {
322 			ret = qbman_swp_acquire(swp, bpid, (void *)bufs,
323 						count - n);
324 		}
325 		/* In case of less than requested number of buffers available
326 		 * in pool, qbman_swp_acquire returns 0
327 		 */
328 		if (ret <= 0) {
329 			DPAA2_MEMPOOL_DP_DEBUG(
330 				"Buffer acquire failed with err code: %d", ret);
331 			/* The API expect the exact number of requested bufs */
332 			/* Releasing all buffers allocated */
333 			rte_dpaa2_mbuf_release(pool, obj_table, bpid,
334 					   bp_info->meta_data_size, n);
335 			return -ENOBUFS;
336 		}
337 		/* assigning mbuf from the acquired objects */
338 		for (i = 0; (i < ret) && bufs[i]; i++) {
339 			DPAA2_MODIFY_IOVA_TO_VADDR(bufs[i], size_t);
340 			obj_table[n] = (struct rte_mbuf *)
341 				       (bufs[i] - bp_info->meta_data_size);
342 			DPAA2_MEMPOOL_DP_DEBUG(
343 				   "Acquired %p address %p from BMAN\n",
344 				   (void *)bufs[i], (void *)obj_table[n]);
345 			n++;
346 		}
347 	}
348 
349 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER
350 	alloc += n;
351 	DPAA2_MEMPOOL_DP_DEBUG("Total = %d , req = %d done = %d\n",
352 			       alloc, count, n);
353 #endif
354 	return 0;
355 }
356 
357 static int
358 rte_hw_mbuf_free_bulk(struct rte_mempool *pool,
359 		  void * const *obj_table, unsigned int n)
360 {
361 	struct dpaa2_bp_info *bp_info;
362 
363 	bp_info = mempool_to_bpinfo(pool);
364 	if (!(bp_info->bp_list)) {
365 		DPAA2_MEMPOOL_ERR("DPAA2 buffer pool not configured");
366 		return -ENOENT;
367 	}
368 	rte_dpaa2_mbuf_release(pool, obj_table, bp_info->bpid,
369 			   bp_info->meta_data_size, n);
370 
371 	return 0;
372 }
373 
374 static unsigned int
375 rte_hw_mbuf_get_count(const struct rte_mempool *mp)
376 {
377 	int ret;
378 	unsigned int num_of_bufs = 0;
379 	struct dpaa2_bp_info *bp_info;
380 	struct dpaa2_dpbp_dev *dpbp_node;
381 
382 	if (!mp || !mp->pool_data) {
383 		DPAA2_MEMPOOL_ERR("Invalid mempool provided");
384 		return 0;
385 	}
386 
387 	bp_info = (struct dpaa2_bp_info *)mp->pool_data;
388 	dpbp_node = bp_info->bp_list->buf_pool.dpbp_node;
389 
390 	ret = dpbp_get_num_free_bufs(&dpbp_node->dpbp, CMD_PRI_LOW,
391 				     dpbp_node->token, &num_of_bufs);
392 	if (ret) {
393 		DPAA2_MEMPOOL_ERR("Unable to obtain free buf count (err=%d)",
394 				  ret);
395 		return 0;
396 	}
397 
398 	DPAA2_MEMPOOL_DP_DEBUG("Free bufs = %u\n", num_of_bufs);
399 
400 	return num_of_bufs;
401 }
402 
403 static int
404 dpaa2_populate(struct rte_mempool *mp, unsigned int max_objs,
405 	      void *vaddr, rte_iova_t paddr, size_t len,
406 	      rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
407 {
408 	/* Insert entry into the PA->VA Table */
409 	dpaax_iova_table_update(paddr, vaddr, len);
410 
411 	return rte_mempool_op_populate_default(mp, max_objs, vaddr, paddr, len,
412 					       obj_cb, obj_cb_arg);
413 }
414 
415 static const struct rte_mempool_ops dpaa2_mpool_ops = {
416 	.name = DPAA2_MEMPOOL_OPS_NAME,
417 	.alloc = rte_hw_mbuf_create_pool,
418 	.free = rte_hw_mbuf_free_pool,
419 	.enqueue = rte_hw_mbuf_free_bulk,
420 	.dequeue = rte_dpaa2_mbuf_alloc_bulk,
421 	.get_count = rte_hw_mbuf_get_count,
422 	.populate = dpaa2_populate,
423 };
424 
425 MEMPOOL_REGISTER_OPS(dpaa2_mpool_ops);
426 
427 RTE_INIT(dpaa2_mempool_init_log)
428 {
429 	dpaa2_logtype_mempool = rte_log_register("mempool.dpaa2");
430 	if (dpaa2_logtype_mempool >= 0)
431 		rte_log_set_level(dpaa2_logtype_mempool, RTE_LOG_NOTICE);
432 }
433