1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
3 */
4
5 #include "roc_api.h"
6 #include "roc_priv.h"
7
8 static roc_npa_lf_init_cb_t lf_init_cb;
9
10 int
roc_npa_lf_init_cb_register(roc_npa_lf_init_cb_t cb)11 roc_npa_lf_init_cb_register(roc_npa_lf_init_cb_t cb)
12 {
13 if (lf_init_cb != NULL)
14 return -EEXIST;
15
16 lf_init_cb = cb;
17 return 0;
18 }
19
20 void
roc_npa_aura_op_range_set(uint64_t aura_handle,uint64_t start_iova,uint64_t end_iova)21 roc_npa_aura_op_range_set(uint64_t aura_handle, uint64_t start_iova,
22 uint64_t end_iova)
23 {
24 const uint64_t start = roc_npa_aura_handle_to_base(aura_handle) +
25 NPA_LF_POOL_OP_PTR_START0;
26 const uint64_t end = roc_npa_aura_handle_to_base(aura_handle) +
27 NPA_LF_POOL_OP_PTR_END0;
28 uint64_t reg = roc_npa_aura_handle_to_aura(aura_handle);
29 struct npa_lf *lf = idev_npa_obj_get();
30 struct npa_aura_lim *lim;
31
32 PLT_ASSERT(lf);
33 lim = lf->aura_lim;
34
35 lim[reg].ptr_start = PLT_MIN(lim[reg].ptr_start, start_iova);
36 lim[reg].ptr_end = PLT_MAX(lim[reg].ptr_end, end_iova);
37
38 roc_store_pair(lim[reg].ptr_start, reg, start);
39 roc_store_pair(lim[reg].ptr_end, reg, end);
40 }
41
42 static int
npa_aura_pool_init(struct mbox * mbox,uint32_t aura_id,struct npa_aura_s * aura,struct npa_pool_s * pool)43 npa_aura_pool_init(struct mbox *mbox, uint32_t aura_id, struct npa_aura_s *aura,
44 struct npa_pool_s *pool)
45 {
46 struct npa_aq_enq_req *aura_init_req, *pool_init_req;
47 struct npa_aq_enq_rsp *aura_init_rsp, *pool_init_rsp;
48 struct mbox_dev *mdev = &mbox->dev[0];
49 int rc = -ENOSPC, off;
50
51 aura_init_req = mbox_alloc_msg_npa_aq_enq(mbox);
52 if (aura_init_req == NULL)
53 return rc;
54 aura_init_req->aura_id = aura_id;
55 aura_init_req->ctype = NPA_AQ_CTYPE_AURA;
56 aura_init_req->op = NPA_AQ_INSTOP_INIT;
57 mbox_memcpy(&aura_init_req->aura, aura, sizeof(*aura));
58
59 pool_init_req = mbox_alloc_msg_npa_aq_enq(mbox);
60 if (pool_init_req == NULL)
61 return rc;
62 pool_init_req->aura_id = aura_id;
63 pool_init_req->ctype = NPA_AQ_CTYPE_POOL;
64 pool_init_req->op = NPA_AQ_INSTOP_INIT;
65 mbox_memcpy(&pool_init_req->pool, pool, sizeof(*pool));
66
67 rc = mbox_process(mbox);
68 if (rc < 0)
69 return rc;
70
71 off = mbox->rx_start +
72 PLT_ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
73 aura_init_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
74 off = mbox->rx_start + aura_init_rsp->hdr.next_msgoff;
75 pool_init_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
76
77 if (aura_init_rsp->hdr.rc == 0 && pool_init_rsp->hdr.rc == 0)
78 return 0;
79 else
80 return NPA_ERR_AURA_POOL_INIT;
81 }
82
83 static int
npa_aura_pool_fini(struct mbox * mbox,uint32_t aura_id,uint64_t aura_handle)84 npa_aura_pool_fini(struct mbox *mbox, uint32_t aura_id, uint64_t aura_handle)
85 {
86 struct npa_aq_enq_req *aura_req, *pool_req;
87 struct npa_aq_enq_rsp *aura_rsp, *pool_rsp;
88 struct mbox_dev *mdev = &mbox->dev[0];
89 struct ndc_sync_op *ndc_req;
90 int rc = -ENOSPC, off;
91 uint64_t ptr;
92
93 /* Procedure for disabling an aura/pool */
94 plt_delay_us(10);
95
96 /* Clear all the pointers from the aura */
97 do {
98 ptr = roc_npa_aura_op_alloc(aura_handle, 0);
99 } while (ptr);
100
101 pool_req = mbox_alloc_msg_npa_aq_enq(mbox);
102 if (pool_req == NULL)
103 return rc;
104 pool_req->aura_id = aura_id;
105 pool_req->ctype = NPA_AQ_CTYPE_POOL;
106 pool_req->op = NPA_AQ_INSTOP_WRITE;
107 pool_req->pool.ena = 0;
108 pool_req->pool_mask.ena = ~pool_req->pool_mask.ena;
109
110 aura_req = mbox_alloc_msg_npa_aq_enq(mbox);
111 if (aura_req == NULL)
112 return rc;
113 aura_req->aura_id = aura_id;
114 aura_req->ctype = NPA_AQ_CTYPE_AURA;
115 aura_req->op = NPA_AQ_INSTOP_WRITE;
116 aura_req->aura.ena = 0;
117 aura_req->aura_mask.ena = ~aura_req->aura_mask.ena;
118
119 rc = mbox_process(mbox);
120 if (rc < 0)
121 return rc;
122
123 off = mbox->rx_start +
124 PLT_ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
125 pool_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
126
127 off = mbox->rx_start + pool_rsp->hdr.next_msgoff;
128 aura_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
129
130 if (aura_rsp->hdr.rc != 0 || pool_rsp->hdr.rc != 0)
131 return NPA_ERR_AURA_POOL_FINI;
132
133 /* Sync NDC-NPA for LF */
134 ndc_req = mbox_alloc_msg_ndc_sync_op(mbox);
135 if (ndc_req == NULL)
136 return -ENOSPC;
137 ndc_req->npa_lf_sync = 1;
138 rc = mbox_process(mbox);
139 if (rc) {
140 plt_err("Error on NDC-NPA LF sync, rc %d", rc);
141 return NPA_ERR_AURA_POOL_FINI;
142 }
143 return 0;
144 }
145
146 int
roc_npa_pool_op_pc_reset(uint64_t aura_handle)147 roc_npa_pool_op_pc_reset(uint64_t aura_handle)
148 {
149 struct npa_lf *lf = idev_npa_obj_get();
150 struct npa_aq_enq_req *pool_req;
151 struct npa_aq_enq_rsp *pool_rsp;
152 struct ndc_sync_op *ndc_req;
153 struct mbox_dev *mdev;
154 int rc = -ENOSPC, off;
155 struct mbox *mbox;
156
157 if (lf == NULL)
158 return NPA_ERR_PARAM;
159
160 mbox = lf->mbox;
161 mdev = &mbox->dev[0];
162 plt_npa_dbg("lf=%p aura_handle=0x%" PRIx64, lf, aura_handle);
163
164 pool_req = mbox_alloc_msg_npa_aq_enq(mbox);
165 if (pool_req == NULL)
166 return rc;
167 pool_req->aura_id = roc_npa_aura_handle_to_aura(aura_handle);
168 pool_req->ctype = NPA_AQ_CTYPE_POOL;
169 pool_req->op = NPA_AQ_INSTOP_WRITE;
170 pool_req->pool.op_pc = 0;
171 pool_req->pool_mask.op_pc = ~pool_req->pool_mask.op_pc;
172
173 rc = mbox_process(mbox);
174 if (rc < 0)
175 return rc;
176
177 off = mbox->rx_start +
178 PLT_ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
179 pool_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
180
181 if (pool_rsp->hdr.rc != 0)
182 return NPA_ERR_AURA_POOL_FINI;
183
184 /* Sync NDC-NPA for LF */
185 ndc_req = mbox_alloc_msg_ndc_sync_op(mbox);
186 if (ndc_req == NULL)
187 return -ENOSPC;
188 ndc_req->npa_lf_sync = 1;
189 rc = mbox_process(mbox);
190 if (rc) {
191 plt_err("Error on NDC-NPA LF sync, rc %d", rc);
192 return NPA_ERR_AURA_POOL_FINI;
193 }
194 return 0;
195 }
196
197 int
roc_npa_aura_drop_set(uint64_t aura_handle,uint64_t limit,bool ena)198 roc_npa_aura_drop_set(uint64_t aura_handle, uint64_t limit, bool ena)
199 {
200 struct npa_aq_enq_req *aura_req;
201 struct npa_lf *lf;
202 int rc;
203
204 lf = idev_npa_obj_get();
205 if (lf == NULL)
206 return NPA_ERR_DEVICE_NOT_BOUNDED;
207
208 aura_req = mbox_alloc_msg_npa_aq_enq(lf->mbox);
209 if (aura_req == NULL)
210 return -ENOMEM;
211 aura_req->aura_id = roc_npa_aura_handle_to_aura(aura_handle);
212 aura_req->ctype = NPA_AQ_CTYPE_AURA;
213 aura_req->op = NPA_AQ_INSTOP_WRITE;
214
215 aura_req->aura.aura_drop_ena = ena;
216 aura_req->aura.aura_drop = limit;
217 aura_req->aura_mask.aura_drop_ena =
218 ~(aura_req->aura_mask.aura_drop_ena);
219 aura_req->aura_mask.aura_drop = ~(aura_req->aura_mask.aura_drop);
220 rc = mbox_process(lf->mbox);
221
222 return rc;
223 }
224
225 static inline char *
npa_stack_memzone_name(struct npa_lf * lf,int pool_id,char * name)226 npa_stack_memzone_name(struct npa_lf *lf, int pool_id, char *name)
227 {
228 snprintf(name, PLT_MEMZONE_NAMESIZE, "roc_npa_stack_%x_%d", lf->pf_func,
229 pool_id);
230 return name;
231 }
232
233 static inline const struct plt_memzone *
npa_stack_dma_alloc(struct npa_lf * lf,char * name,int pool_id,size_t size)234 npa_stack_dma_alloc(struct npa_lf *lf, char *name, int pool_id, size_t size)
235 {
236 const char *mz_name = npa_stack_memzone_name(lf, pool_id, name);
237 size = PLT_ALIGN_CEIL(size, ROC_ALIGN);
238
239 return plt_memzone_reserve_aligned(mz_name, size, 0, ROC_ALIGN);
240 }
241
242 static inline int
npa_stack_dma_free(struct npa_lf * lf,char * name,int pool_id)243 npa_stack_dma_free(struct npa_lf *lf, char *name, int pool_id)
244 {
245 const struct plt_memzone *mz;
246
247 mz = plt_memzone_lookup(npa_stack_memzone_name(lf, pool_id, name));
248 if (mz == NULL)
249 return NPA_ERR_PARAM;
250
251 return plt_memzone_free(mz);
252 }
253
254 static inline int
bitmap_ctzll(uint64_t slab)255 bitmap_ctzll(uint64_t slab)
256 {
257 if (slab == 0)
258 return 0;
259
260 return __builtin_ctzll(slab);
261 }
262
263 static int
npa_aura_pool_pair_alloc(struct npa_lf * lf,const uint32_t block_size,const uint32_t block_count,struct npa_aura_s * aura,struct npa_pool_s * pool,uint64_t * aura_handle)264 npa_aura_pool_pair_alloc(struct npa_lf *lf, const uint32_t block_size,
265 const uint32_t block_count, struct npa_aura_s *aura,
266 struct npa_pool_s *pool, uint64_t *aura_handle)
267 {
268 int rc, aura_id, pool_id, stack_size, alloc_size;
269 char name[PLT_MEMZONE_NAMESIZE];
270 const struct plt_memzone *mz;
271 uint64_t slab;
272 uint32_t pos;
273
274 /* Sanity check */
275 if (!lf || !block_size || !block_count || !pool || !aura ||
276 !aura_handle)
277 return NPA_ERR_PARAM;
278
279 /* Block size should be cache line aligned and in range of 128B-128KB */
280 if (block_size % ROC_ALIGN || block_size < 128 ||
281 block_size > ROC_NPA_MAX_BLOCK_SZ)
282 return NPA_ERR_INVALID_BLOCK_SZ;
283
284 pos = 0;
285 slab = 0;
286 /* Scan from the beginning */
287 plt_bitmap_scan_init(lf->npa_bmp);
288 /* Scan bitmap to get the free pool */
289 rc = plt_bitmap_scan(lf->npa_bmp, &pos, &slab);
290 /* Empty bitmap */
291 if (rc == 0) {
292 plt_err("Mempools exhausted");
293 return NPA_ERR_AURA_ID_ALLOC;
294 }
295
296 /* Get aura_id from resource bitmap */
297 aura_id = pos + bitmap_ctzll(slab);
298 /* Mark pool as reserved */
299 plt_bitmap_clear(lf->npa_bmp, aura_id);
300
301 /* Configuration based on each aura has separate pool(aura-pool pair) */
302 pool_id = aura_id;
303 rc = (aura_id < 0 || pool_id >= (int)lf->nr_pools ||
304 aura_id >= (int)BIT_ULL(6 + lf->aura_sz)) ?
305 NPA_ERR_AURA_ID_ALLOC :
306 0;
307 if (rc)
308 goto exit;
309
310 /* Allocate stack memory */
311 stack_size = (block_count + lf->stack_pg_ptrs - 1) / lf->stack_pg_ptrs;
312 alloc_size = stack_size * lf->stack_pg_bytes;
313
314 mz = npa_stack_dma_alloc(lf, name, pool_id, alloc_size);
315 if (mz == NULL) {
316 rc = NPA_ERR_ALLOC;
317 goto aura_res_put;
318 }
319
320 /* Update aura fields */
321 aura->pool_addr = pool_id; /* AF will translate to associated poolctx */
322 aura->ena = 1;
323 aura->shift = plt_log2_u32(block_count);
324 aura->shift = aura->shift < 8 ? 0 : aura->shift - 8;
325 aura->limit = block_count;
326 aura->pool_caching = 1;
327 aura->err_int_ena = BIT(NPA_AURA_ERR_INT_AURA_ADD_OVER);
328 aura->err_int_ena |= BIT(NPA_AURA_ERR_INT_AURA_ADD_UNDER);
329 aura->err_int_ena |= BIT(NPA_AURA_ERR_INT_AURA_FREE_UNDER);
330 aura->err_int_ena |= BIT(NPA_AURA_ERR_INT_POOL_DIS);
331 aura->avg_con = 0;
332 /* Many to one reduction */
333 aura->err_qint_idx = aura_id % lf->qints;
334
335 /* Update pool fields */
336 pool->stack_base = mz->iova;
337 pool->ena = 1;
338 pool->buf_size = block_size / ROC_ALIGN;
339 pool->stack_max_pages = stack_size;
340 pool->shift = plt_log2_u32(block_count);
341 pool->shift = pool->shift < 8 ? 0 : pool->shift - 8;
342 pool->ptr_start = 0;
343 pool->ptr_end = ~0;
344 pool->stack_caching = 1;
345 pool->err_int_ena = BIT(NPA_POOL_ERR_INT_OVFLS);
346 pool->err_int_ena |= BIT(NPA_POOL_ERR_INT_RANGE);
347 pool->err_int_ena |= BIT(NPA_POOL_ERR_INT_PERR);
348 pool->avg_con = 0;
349
350 /* Many to one reduction */
351 pool->err_qint_idx = pool_id % lf->qints;
352
353 /* Issue AURA_INIT and POOL_INIT op */
354 rc = npa_aura_pool_init(lf->mbox, aura_id, aura, pool);
355 if (rc)
356 goto stack_mem_free;
357
358 *aura_handle = roc_npa_aura_handle_gen(aura_id, lf->base);
359 /* Update aura count */
360 roc_npa_aura_op_cnt_set(*aura_handle, 0, block_count);
361 /* Read it back to make sure aura count is updated */
362 roc_npa_aura_op_cnt_get(*aura_handle);
363
364 return 0;
365
366 stack_mem_free:
367 plt_memzone_free(mz);
368 aura_res_put:
369 plt_bitmap_set(lf->npa_bmp, aura_id);
370 exit:
371 return rc;
372 }
373
374 int
roc_npa_pool_create(uint64_t * aura_handle,uint32_t block_size,uint32_t block_count,struct npa_aura_s * aura,struct npa_pool_s * pool)375 roc_npa_pool_create(uint64_t *aura_handle, uint32_t block_size,
376 uint32_t block_count, struct npa_aura_s *aura,
377 struct npa_pool_s *pool)
378 {
379 struct npa_aura_s defaura;
380 struct npa_pool_s defpool;
381 struct idev_cfg *idev;
382 struct npa_lf *lf;
383 int rc;
384
385 lf = idev_npa_obj_get();
386 if (lf == NULL) {
387 rc = NPA_ERR_DEVICE_NOT_BOUNDED;
388 goto error;
389 }
390
391 idev = idev_get_cfg();
392 if (idev == NULL) {
393 rc = NPA_ERR_ALLOC;
394 goto error;
395 }
396
397 if (aura == NULL) {
398 memset(&defaura, 0, sizeof(struct npa_aura_s));
399 aura = &defaura;
400 }
401 if (pool == NULL) {
402 memset(&defpool, 0, sizeof(struct npa_pool_s));
403 defpool.nat_align = 1;
404 defpool.buf_offset = 1;
405 pool = &defpool;
406 }
407
408 rc = npa_aura_pool_pair_alloc(lf, block_size, block_count, aura, pool,
409 aura_handle);
410 if (rc) {
411 plt_err("Failed to alloc pool or aura rc=%d", rc);
412 goto error;
413 }
414
415 plt_npa_dbg("lf=%p block_sz=%d block_count=%d aura_handle=0x%" PRIx64,
416 lf, block_size, block_count, *aura_handle);
417
418 /* Just hold the reference of the object */
419 __atomic_fetch_add(&idev->npa_refcnt, 1, __ATOMIC_SEQ_CST);
420 error:
421 return rc;
422 }
423
424 int
roc_npa_aura_limit_modify(uint64_t aura_handle,uint16_t aura_limit)425 roc_npa_aura_limit_modify(uint64_t aura_handle, uint16_t aura_limit)
426 {
427 struct npa_aq_enq_req *aura_req;
428 struct npa_lf *lf;
429 int rc;
430
431 lf = idev_npa_obj_get();
432 if (lf == NULL)
433 return NPA_ERR_DEVICE_NOT_BOUNDED;
434
435 aura_req = mbox_alloc_msg_npa_aq_enq(lf->mbox);
436 if (aura_req == NULL)
437 return -ENOMEM;
438 aura_req->aura_id = roc_npa_aura_handle_to_aura(aura_handle);
439 aura_req->ctype = NPA_AQ_CTYPE_AURA;
440 aura_req->op = NPA_AQ_INSTOP_WRITE;
441
442 aura_req->aura.limit = aura_limit;
443 aura_req->aura_mask.limit = ~(aura_req->aura_mask.limit);
444 rc = mbox_process(lf->mbox);
445
446 return rc;
447 }
448
449 static int
npa_aura_pool_pair_free(struct npa_lf * lf,uint64_t aura_handle)450 npa_aura_pool_pair_free(struct npa_lf *lf, uint64_t aura_handle)
451 {
452 char name[PLT_MEMZONE_NAMESIZE];
453 int aura_id, pool_id, rc;
454
455 if (!lf || !aura_handle)
456 return NPA_ERR_PARAM;
457
458 aura_id = roc_npa_aura_handle_to_aura(aura_handle);
459 pool_id = aura_id;
460 rc = npa_aura_pool_fini(lf->mbox, aura_id, aura_handle);
461 rc |= npa_stack_dma_free(lf, name, pool_id);
462
463 plt_bitmap_set(lf->npa_bmp, aura_id);
464
465 return rc;
466 }
467
468 int
roc_npa_pool_destroy(uint64_t aura_handle)469 roc_npa_pool_destroy(uint64_t aura_handle)
470 {
471 struct npa_lf *lf = idev_npa_obj_get();
472 int rc = 0;
473
474 plt_npa_dbg("lf=%p aura_handle=0x%" PRIx64, lf, aura_handle);
475 rc = npa_aura_pool_pair_free(lf, aura_handle);
476 if (rc)
477 plt_err("Failed to destroy pool or aura rc=%d", rc);
478
479 /* Release the reference of npa */
480 rc |= npa_lf_fini();
481 return rc;
482 }
483
484 int
roc_npa_pool_range_update_check(uint64_t aura_handle)485 roc_npa_pool_range_update_check(uint64_t aura_handle)
486 {
487 uint64_t aura_id = roc_npa_aura_handle_to_aura(aura_handle);
488 struct npa_lf *lf;
489 struct npa_aura_lim *lim;
490 __io struct npa_pool_s *pool;
491 struct npa_aq_enq_req *req;
492 struct npa_aq_enq_rsp *rsp;
493 int rc;
494
495 lf = idev_npa_obj_get();
496 if (lf == NULL)
497 return NPA_ERR_PARAM;
498
499 lim = lf->aura_lim;
500
501 req = mbox_alloc_msg_npa_aq_enq(lf->mbox);
502 if (req == NULL)
503 return -ENOSPC;
504
505 req->aura_id = aura_id;
506 req->ctype = NPA_AQ_CTYPE_POOL;
507 req->op = NPA_AQ_INSTOP_READ;
508
509 rc = mbox_process_msg(lf->mbox, (void *)&rsp);
510 if (rc) {
511 plt_err("Failed to get pool(0x%" PRIx64 ") context", aura_id);
512 return rc;
513 }
514
515 pool = &rsp->pool;
516 if (lim[aura_id].ptr_start != pool->ptr_start ||
517 lim[aura_id].ptr_end != pool->ptr_end) {
518 plt_err("Range update failed on pool(0x%" PRIx64 ")", aura_id);
519 return NPA_ERR_PARAM;
520 }
521
522 return 0;
523 }
524
525 static inline int
npa_attach(struct mbox * mbox)526 npa_attach(struct mbox *mbox)
527 {
528 struct rsrc_attach_req *req;
529
530 req = mbox_alloc_msg_attach_resources(mbox);
531 if (req == NULL)
532 return -ENOSPC;
533 req->modify = true;
534 req->npalf = true;
535
536 return mbox_process(mbox);
537 }
538
539 static inline int
npa_detach(struct mbox * mbox)540 npa_detach(struct mbox *mbox)
541 {
542 struct rsrc_detach_req *req;
543
544 req = mbox_alloc_msg_detach_resources(mbox);
545 if (req == NULL)
546 return -ENOSPC;
547 req->partial = true;
548 req->npalf = true;
549
550 return mbox_process(mbox);
551 }
552
553 static inline int
npa_get_msix_offset(struct mbox * mbox,uint16_t * npa_msixoff)554 npa_get_msix_offset(struct mbox *mbox, uint16_t *npa_msixoff)
555 {
556 struct msix_offset_rsp *msix_rsp;
557 int rc;
558
559 /* Get NPA MSIX vector offsets */
560 mbox_alloc_msg_msix_offset(mbox);
561 rc = mbox_process_msg(mbox, (void *)&msix_rsp);
562 if (rc == 0)
563 *npa_msixoff = msix_rsp->npa_msixoff;
564
565 return rc;
566 }
567
568 static inline int
npa_lf_alloc(struct npa_lf * lf)569 npa_lf_alloc(struct npa_lf *lf)
570 {
571 struct mbox *mbox = lf->mbox;
572 struct npa_lf_alloc_req *req;
573 struct npa_lf_alloc_rsp *rsp;
574 int rc;
575
576 req = mbox_alloc_msg_npa_lf_alloc(mbox);
577 if (req == NULL)
578 return -ENOSPC;
579 req->aura_sz = lf->aura_sz;
580 req->nr_pools = lf->nr_pools;
581
582 rc = mbox_process_msg(mbox, (void *)&rsp);
583 if (rc)
584 return NPA_ERR_ALLOC;
585
586 lf->stack_pg_ptrs = rsp->stack_pg_ptrs;
587 lf->stack_pg_bytes = rsp->stack_pg_bytes;
588 lf->qints = rsp->qints;
589
590 return 0;
591 }
592
593 static int
npa_lf_free(struct mbox * mbox)594 npa_lf_free(struct mbox *mbox)
595 {
596 mbox_alloc_msg_npa_lf_free(mbox);
597 return mbox_process(mbox);
598 }
599
600 static inline uint32_t
aura_size_to_u32(uint8_t val)601 aura_size_to_u32(uint8_t val)
602 {
603 if (val == NPA_AURA_SZ_0)
604 return 128;
605 if (val >= NPA_AURA_SZ_MAX)
606 return BIT_ULL(20);
607
608 return 1 << (val + 6);
609 }
610
611 static inline void
pool_count_aura_sz_get(uint32_t * nr_pools,uint8_t * aura_sz)612 pool_count_aura_sz_get(uint32_t *nr_pools, uint8_t *aura_sz)
613 {
614 uint32_t val;
615
616 val = roc_idev_npa_maxpools_get();
617 if (val < aura_size_to_u32(NPA_AURA_SZ_128))
618 val = 128;
619 if (val > aura_size_to_u32(NPA_AURA_SZ_1M))
620 val = BIT_ULL(20);
621
622 roc_idev_npa_maxpools_set(val);
623 *nr_pools = val;
624 *aura_sz = plt_log2_u32(val) - 6;
625 }
626
627 static int
npa_dev_init(struct npa_lf * lf,uintptr_t base,struct mbox * mbox)628 npa_dev_init(struct npa_lf *lf, uintptr_t base, struct mbox *mbox)
629 {
630 uint32_t i, bmp_sz, nr_pools;
631 uint8_t aura_sz;
632 int rc;
633
634 /* Sanity checks */
635 if (!lf || !base || !mbox)
636 return NPA_ERR_PARAM;
637
638 if (base & ROC_AURA_ID_MASK)
639 return NPA_ERR_BASE_INVALID;
640
641 pool_count_aura_sz_get(&nr_pools, &aura_sz);
642 if (aura_sz == NPA_AURA_SZ_0 || aura_sz >= NPA_AURA_SZ_MAX)
643 return NPA_ERR_PARAM;
644
645 memset(lf, 0x0, sizeof(*lf));
646 lf->base = base;
647 lf->aura_sz = aura_sz;
648 lf->nr_pools = nr_pools;
649 lf->mbox = mbox;
650
651 rc = npa_lf_alloc(lf);
652 if (rc)
653 goto exit;
654
655 bmp_sz = plt_bitmap_get_memory_footprint(nr_pools);
656
657 /* Allocate memory for bitmap */
658 lf->npa_bmp_mem = plt_zmalloc(bmp_sz, ROC_ALIGN);
659 if (lf->npa_bmp_mem == NULL) {
660 rc = NPA_ERR_ALLOC;
661 goto lf_free;
662 }
663
664 /* Initialize pool resource bitmap array */
665 lf->npa_bmp = plt_bitmap_init(nr_pools, lf->npa_bmp_mem, bmp_sz);
666 if (lf->npa_bmp == NULL) {
667 rc = NPA_ERR_PARAM;
668 goto bmap_mem_free;
669 }
670
671 /* Mark all pools available */
672 for (i = 0; i < nr_pools; i++)
673 plt_bitmap_set(lf->npa_bmp, i);
674
675 /* Allocate memory for qint context */
676 lf->npa_qint_mem = plt_zmalloc(sizeof(struct npa_qint) * nr_pools, 0);
677 if (lf->npa_qint_mem == NULL) {
678 rc = NPA_ERR_ALLOC;
679 goto bmap_free;
680 }
681
682 /* Allocate memory for nap_aura_lim memory */
683 lf->aura_lim = plt_zmalloc(sizeof(struct npa_aura_lim) * nr_pools, 0);
684 if (lf->aura_lim == NULL) {
685 rc = NPA_ERR_ALLOC;
686 goto qint_free;
687 }
688
689 /* Init aura start & end limits */
690 for (i = 0; i < nr_pools; i++) {
691 lf->aura_lim[i].ptr_start = UINT64_MAX;
692 lf->aura_lim[i].ptr_end = 0x0ull;
693 }
694
695 return 0;
696
697 qint_free:
698 plt_free(lf->npa_qint_mem);
699 bmap_free:
700 plt_bitmap_free(lf->npa_bmp);
701 bmap_mem_free:
702 plt_free(lf->npa_bmp_mem);
703 lf_free:
704 npa_lf_free(lf->mbox);
705 exit:
706 return rc;
707 }
708
709 static int
npa_dev_fini(struct npa_lf * lf)710 npa_dev_fini(struct npa_lf *lf)
711 {
712 if (!lf)
713 return NPA_ERR_PARAM;
714
715 plt_free(lf->aura_lim);
716 plt_free(lf->npa_qint_mem);
717 plt_bitmap_free(lf->npa_bmp);
718 plt_free(lf->npa_bmp_mem);
719
720 return npa_lf_free(lf->mbox);
721 }
722
723 int
npa_lf_init(struct dev * dev,struct plt_pci_device * pci_dev)724 npa_lf_init(struct dev *dev, struct plt_pci_device *pci_dev)
725 {
726 struct idev_cfg *idev;
727 uint16_t npa_msixoff;
728 struct npa_lf *lf;
729 int rc;
730
731 idev = idev_get_cfg();
732 if (idev == NULL)
733 return NPA_ERR_ALLOC;
734
735 /* Not the first PCI device */
736 if (__atomic_fetch_add(&idev->npa_refcnt, 1, __ATOMIC_SEQ_CST) != 0)
737 return 0;
738
739 if (lf_init_cb) {
740 rc = (*lf_init_cb)(pci_dev);
741 if (rc)
742 goto fail;
743 }
744
745 rc = npa_attach(dev->mbox);
746 if (rc)
747 goto fail;
748
749 rc = npa_get_msix_offset(dev->mbox, &npa_msixoff);
750 if (rc)
751 goto npa_detach;
752
753 lf = &dev->npa;
754 rc = npa_dev_init(lf, dev->bar2 + (RVU_BLOCK_ADDR_NPA << 20),
755 dev->mbox);
756 if (rc)
757 goto npa_detach;
758
759 lf->pf_func = dev->pf_func;
760 lf->npa_msixoff = npa_msixoff;
761 lf->intr_handle = pci_dev->intr_handle;
762 lf->pci_dev = pci_dev;
763
764 idev->npa_pf_func = dev->pf_func;
765 idev->npa = lf;
766 plt_wmb();
767
768 rc = npa_register_irqs(lf);
769 if (rc)
770 goto npa_fini;
771
772 plt_npa_dbg("npa=%p max_pools=%d pf_func=0x%x msix=0x%x", lf,
773 roc_idev_npa_maxpools_get(), lf->pf_func, npa_msixoff);
774
775 return 0;
776
777 npa_fini:
778 npa_dev_fini(idev->npa);
779 npa_detach:
780 npa_detach(dev->mbox);
781 fail:
782 __atomic_fetch_sub(&idev->npa_refcnt, 1, __ATOMIC_SEQ_CST);
783 return rc;
784 }
785
786 int
npa_lf_fini(void)787 npa_lf_fini(void)
788 {
789 struct idev_cfg *idev;
790 int rc = 0;
791
792 idev = idev_get_cfg();
793 if (idev == NULL)
794 return NPA_ERR_ALLOC;
795
796 /* Not the last PCI device */
797 if (__atomic_sub_fetch(&idev->npa_refcnt, 1, __ATOMIC_SEQ_CST) != 0)
798 return 0;
799
800 npa_unregister_irqs(idev->npa);
801 rc |= npa_dev_fini(idev->npa);
802 rc |= npa_detach(idev->npa->mbox);
803 idev_set_defaults(idev);
804
805 return rc;
806 }
807
808 int
roc_npa_dev_init(struct roc_npa * roc_npa)809 roc_npa_dev_init(struct roc_npa *roc_npa)
810 {
811 struct plt_pci_device *pci_dev;
812 struct npa *npa;
813 struct dev *dev;
814 int rc;
815
816 if (roc_npa == NULL || roc_npa->pci_dev == NULL)
817 return NPA_ERR_PARAM;
818
819 PLT_STATIC_ASSERT(sizeof(struct npa) <= ROC_NPA_MEM_SZ);
820 npa = roc_npa_to_npa_priv(roc_npa);
821 memset(npa, 0, sizeof(*npa));
822 pci_dev = roc_npa->pci_dev;
823 dev = &npa->dev;
824
825 /* Initialize device */
826 rc = dev_init(dev, pci_dev);
827 if (rc) {
828 plt_err("Failed to init roc device");
829 goto fail;
830 }
831
832 npa->pci_dev = pci_dev;
833 dev->drv_inited = true;
834 fail:
835 return rc;
836 }
837
838 int
roc_npa_dev_fini(struct roc_npa * roc_npa)839 roc_npa_dev_fini(struct roc_npa *roc_npa)
840 {
841 struct npa *npa = roc_npa_to_npa_priv(roc_npa);
842
843 if (npa == NULL)
844 return NPA_ERR_PARAM;
845
846 npa->dev.drv_inited = false;
847 return dev_fini(&npa->dev, npa->pci_dev);
848 }
849