1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Cavium, Inc
3 */
4
5 #include <stdlib.h>
6 #include <string.h>
7 #include <stdbool.h>
8 #include <stdio.h>
9 #include <unistd.h>
10 #include <fcntl.h>
11 #include <errno.h>
12 #include <sys/mman.h>
13
14 #include <rte_atomic.h>
15 #include <rte_eal.h>
16 #include <rte_bus_pci.h>
17 #include <rte_errno.h>
18 #include <rte_memory.h>
19 #include <rte_malloc.h>
20 #include <rte_spinlock.h>
21 #include <rte_mbuf.h>
22
23 #include "octeontx_mbox.h"
24 #include "octeontx_fpavf.h"
25
26 /* FPA Mbox Message */
27 #define IDENTIFY 0x0
28
29 #define FPA_CONFIGSET 0x1
30 #define FPA_CONFIGGET 0x2
31 #define FPA_START_COUNT 0x3
32 #define FPA_STOP_COUNT 0x4
33 #define FPA_ATTACHAURA 0x5
34 #define FPA_DETACHAURA 0x6
35 #define FPA_SETAURALVL 0x7
36 #define FPA_GETAURALVL 0x8
37
38 #define FPA_COPROC 0x1
39
40 /* fpa mbox struct */
41 struct octeontx_mbox_fpa_cfg {
42 int aid;
43 uint64_t pool_cfg;
44 uint64_t pool_stack_base;
45 uint64_t pool_stack_end;
46 uint64_t aura_cfg;
47 };
48
49 struct __rte_packed gen_req {
50 uint32_t value;
51 };
52
53 struct __rte_packed idn_req {
54 uint8_t domain_id;
55 };
56
57 struct __rte_packed gen_resp {
58 uint16_t domain_id;
59 uint16_t vfid;
60 };
61
62 struct __rte_packed dcfg_resp {
63 uint8_t sso_count;
64 uint8_t ssow_count;
65 uint8_t fpa_count;
66 uint8_t pko_count;
67 uint8_t tim_count;
68 uint8_t net_port_count;
69 uint8_t virt_port_count;
70 };
71
72 #define FPA_MAX_POOL 32
73 #define FPA_PF_PAGE_SZ 4096
74
75 #define FPA_LN_SIZE 128
76 #define FPA_ROUND_UP(x, size) \
77 ((((unsigned long)(x)) + size-1) & (~(size-1)))
78 #define FPA_OBJSZ_2_CACHE_LINE(sz) (((sz) + RTE_CACHE_LINE_MASK) >> 7)
79 #define FPA_CACHE_LINE_2_OBJSZ(sz) ((sz) << 7)
80
81 #define POOL_ENA (0x1 << 0)
82 #define POOL_DIS (0x0 << 0)
83 #define POOL_SET_NAT_ALIGN (0x1 << 1)
84 #define POOL_DIS_NAT_ALIGN (0x0 << 1)
85 #define POOL_STYPE(x) (((x) & 0x1) << 2)
86 #define POOL_LTYPE(x) (((x) & 0x3) << 3)
87 #define POOL_BUF_OFFSET(x) (((x) & 0x7fffULL) << 16)
88 #define POOL_BUF_SIZE(x) (((x) & 0x7ffULL) << 32)
89
90 struct fpavf_res {
91 void *pool_stack_base;
92 void *bar0;
93 uint64_t stack_ln_ptr;
94 uint16_t domain_id;
95 uint16_t vf_id; /* gpool_id */
96 uint16_t sz128; /* Block size in cache lines */
97 bool is_inuse;
98 };
99
100 struct octeontx_fpadev {
101 rte_spinlock_t lock;
102 uint8_t total_gpool_cnt;
103 struct fpavf_res pool[FPA_VF_MAX];
104 };
105
106 static struct octeontx_fpadev fpadev;
107
108 RTE_LOG_REGISTER(octeontx_logtype_fpavf, pmd.mempool.octeontx, NOTICE);
109
110 /* lock is taken by caller */
111 static int
octeontx_fpa_gpool_alloc(unsigned int object_size)112 octeontx_fpa_gpool_alloc(unsigned int object_size)
113 {
114 uint16_t global_domain = octeontx_get_global_domain();
115 struct fpavf_res *res = NULL;
116 unsigned int sz128;
117 int i;
118
119 sz128 = FPA_OBJSZ_2_CACHE_LINE(object_size);
120
121 for (i = 0; i < FPA_VF_MAX; i++) {
122
123 /* Skip VF that is not mapped Or _inuse */
124 if ((fpadev.pool[i].bar0 == NULL) ||
125 (fpadev.pool[i].is_inuse == true) ||
126 (fpadev.pool[i].domain_id != global_domain))
127 continue;
128
129 res = &fpadev.pool[i];
130
131 RTE_ASSERT(res->domain_id != (uint16_t)~0);
132 RTE_ASSERT(res->vf_id != (uint16_t)~0);
133 RTE_ASSERT(res->stack_ln_ptr != 0);
134
135 if (res->sz128 == 0) {
136 res->sz128 = sz128;
137 fpavf_log_dbg("gpool %d blk_sz %d\n", res->vf_id,
138 sz128);
139
140 return res->vf_id;
141 }
142 }
143
144 return -ENOSPC;
145 }
146
147 static __rte_always_inline struct fpavf_res *
octeontx_get_fpavf(uint16_t gpool)148 octeontx_get_fpavf(uint16_t gpool)
149 {
150 uint16_t global_domain = octeontx_get_global_domain();
151 int i;
152
153 for (i = 0; i < FPA_VF_MAX; i++) {
154 if (fpadev.pool[i].domain_id != global_domain)
155 continue;
156 if (fpadev.pool[i].vf_id != gpool)
157 continue;
158
159 return &fpadev.pool[i];
160 }
161
162 return NULL;
163 }
164
165 /* lock is taken by caller */
166 static __rte_always_inline uintptr_t
octeontx_fpa_gpool2handle(uint16_t gpool)167 octeontx_fpa_gpool2handle(uint16_t gpool)
168 {
169 struct fpavf_res *res = NULL;
170
171 RTE_ASSERT(gpool < FPA_VF_MAX);
172 res = octeontx_get_fpavf(gpool);
173 if (res == NULL)
174 return 0;
175
176 return (uintptr_t)res->bar0 | gpool;
177 }
178
179 static __rte_always_inline bool
octeontx_fpa_handle_valid(uintptr_t handle)180 octeontx_fpa_handle_valid(uintptr_t handle)
181 {
182 struct fpavf_res *res = NULL;
183 uint8_t gpool;
184 int i;
185 bool ret = false;
186
187 if (unlikely(!handle))
188 return ret;
189
190 /* get the gpool */
191 gpool = octeontx_fpa_bufpool_gpool(handle);
192
193 /* get the bar address */
194 handle &= ~(uint64_t)FPA_GPOOL_MASK;
195 for (i = 0; i < FPA_VF_MAX; i++) {
196 if ((uintptr_t)fpadev.pool[i].bar0 != handle)
197 continue;
198
199 /* validate gpool */
200 if (gpool != fpadev.pool[i].vf_id)
201 return false;
202
203 res = &fpadev.pool[i];
204
205 if (res->sz128 == 0 || res->domain_id == (uint16_t)~0 ||
206 res->stack_ln_ptr == 0)
207 ret = false;
208 else
209 ret = true;
210 break;
211 }
212
213 return ret;
214 }
215
216 static int
octeontx_fpapf_pool_setup(unsigned int gpool,unsigned int buf_size,signed short buf_offset,unsigned int max_buf_count)217 octeontx_fpapf_pool_setup(unsigned int gpool, unsigned int buf_size,
218 signed short buf_offset, unsigned int max_buf_count)
219 {
220 void *memptr = NULL;
221 rte_iova_t phys_addr;
222 unsigned int memsz;
223 struct fpavf_res *fpa = NULL;
224 uint64_t reg;
225 struct octeontx_mbox_hdr hdr;
226 struct dcfg_resp resp;
227 struct octeontx_mbox_fpa_cfg cfg;
228 int ret = -1;
229
230 fpa = octeontx_get_fpavf(gpool);
231 if (fpa == NULL)
232 return -EINVAL;
233
234 memsz = FPA_ROUND_UP(max_buf_count / fpa->stack_ln_ptr, FPA_LN_SIZE) *
235 FPA_LN_SIZE;
236
237 /* Round-up to page size */
238 memsz = (memsz + FPA_PF_PAGE_SZ - 1) & ~(uintptr_t)(FPA_PF_PAGE_SZ-1);
239 memptr = rte_malloc(NULL, memsz, RTE_CACHE_LINE_SIZE);
240 if (memptr == NULL) {
241 ret = -ENOMEM;
242 goto err;
243 }
244
245 /* Configure stack */
246 fpa->pool_stack_base = memptr;
247 phys_addr = rte_malloc_virt2iova(memptr);
248
249 buf_size /= FPA_LN_SIZE;
250
251 /* POOL setup */
252 hdr.coproc = FPA_COPROC;
253 hdr.msg = FPA_CONFIGSET;
254 hdr.vfid = fpa->vf_id;
255 hdr.res_code = 0;
256
257 buf_offset /= FPA_LN_SIZE;
258 reg = POOL_BUF_SIZE(buf_size) | POOL_BUF_OFFSET(buf_offset) |
259 POOL_LTYPE(0x2) | POOL_STYPE(0) | POOL_SET_NAT_ALIGN |
260 POOL_ENA;
261
262 cfg.aid = 0;
263 cfg.pool_cfg = reg;
264 cfg.pool_stack_base = phys_addr;
265 cfg.pool_stack_end = phys_addr + memsz;
266 cfg.aura_cfg = (1 << 9);
267
268 ret = octeontx_mbox_send(&hdr, &cfg,
269 sizeof(struct octeontx_mbox_fpa_cfg),
270 &resp, sizeof(resp));
271 if (ret < 0) {
272 ret = -EACCES;
273 goto err;
274 }
275
276 fpavf_log_dbg(" vfid %d gpool %d aid %d pool_cfg 0x%x pool_stack_base %" PRIx64 " pool_stack_end %" PRIx64" aura_cfg %" PRIx64 "\n",
277 fpa->vf_id, gpool, cfg.aid, (unsigned int)cfg.pool_cfg,
278 cfg.pool_stack_base, cfg.pool_stack_end, cfg.aura_cfg);
279
280 /* Now pool is in_use */
281 fpa->is_inuse = true;
282
283 err:
284 if (ret < 0)
285 rte_free(memptr);
286
287 return ret;
288 }
289
290 static int
octeontx_fpapf_pool_destroy(unsigned int gpool_index)291 octeontx_fpapf_pool_destroy(unsigned int gpool_index)
292 {
293 struct octeontx_mbox_hdr hdr;
294 struct dcfg_resp resp;
295 struct octeontx_mbox_fpa_cfg cfg;
296 struct fpavf_res *fpa = NULL;
297 int ret = -1;
298
299 fpa = octeontx_get_fpavf(gpool_index);
300 if (fpa == NULL)
301 return -EINVAL;
302
303 hdr.coproc = FPA_COPROC;
304 hdr.msg = FPA_CONFIGSET;
305 hdr.vfid = fpa->vf_id;
306 hdr.res_code = 0;
307
308 /* reset and free the pool */
309 cfg.aid = 0;
310 cfg.pool_cfg = 0;
311 cfg.pool_stack_base = 0;
312 cfg.pool_stack_end = 0;
313 cfg.aura_cfg = 0;
314
315 ret = octeontx_mbox_send(&hdr, &cfg,
316 sizeof(struct octeontx_mbox_fpa_cfg),
317 &resp, sizeof(resp));
318 if (ret < 0) {
319 ret = -EACCES;
320 goto err;
321 }
322
323 ret = 0;
324 err:
325 /* anycase free pool stack memory */
326 rte_free(fpa->pool_stack_base);
327 fpa->pool_stack_base = NULL;
328 return ret;
329 }
330
331 static int
octeontx_fpapf_aura_attach(unsigned int gpool_index)332 octeontx_fpapf_aura_attach(unsigned int gpool_index)
333 {
334 struct octeontx_mbox_hdr hdr;
335 struct dcfg_resp resp;
336 struct octeontx_mbox_fpa_cfg cfg;
337 int ret = 0;
338
339 if (gpool_index >= FPA_MAX_POOL) {
340 ret = -EINVAL;
341 goto err;
342 }
343 hdr.coproc = FPA_COPROC;
344 hdr.msg = FPA_ATTACHAURA;
345 hdr.vfid = gpool_index;
346 hdr.res_code = 0;
347 memset(&cfg, 0x0, sizeof(struct octeontx_mbox_fpa_cfg));
348 cfg.aid = 0;
349
350 ret = octeontx_mbox_send(&hdr, &cfg,
351 sizeof(struct octeontx_mbox_fpa_cfg),
352 &resp, sizeof(resp));
353 if (ret < 0) {
354 fpavf_log_err("Could not attach fpa ");
355 fpavf_log_err("aura %d to pool %d. Err=%d. FuncErr=%d\n",
356 FPA_AURA_IDX(gpool_index), gpool_index, ret,
357 hdr.res_code);
358 ret = -EACCES;
359 goto err;
360 }
361 err:
362 return ret;
363 }
364
365 static int
octeontx_fpapf_aura_detach(unsigned int gpool_index)366 octeontx_fpapf_aura_detach(unsigned int gpool_index)
367 {
368 struct octeontx_mbox_fpa_cfg cfg = {0};
369 struct octeontx_mbox_hdr hdr = {0};
370 int ret = 0;
371
372 if (gpool_index >= FPA_MAX_POOL) {
373 ret = -EINVAL;
374 goto err;
375 }
376
377 cfg.aid = 0;
378 hdr.coproc = FPA_COPROC;
379 hdr.msg = FPA_DETACHAURA;
380 hdr.vfid = gpool_index;
381 ret = octeontx_mbox_send(&hdr, &cfg, sizeof(cfg), NULL, 0);
382 if (ret < 0) {
383 fpavf_log_err("Couldn't detach FPA aura %d Err=%d FuncErr=%d\n",
384 FPA_AURA_IDX(gpool_index), ret,
385 hdr.res_code);
386 ret = -EINVAL;
387 }
388
389 err:
390 return ret;
391 }
392
393 int
octeontx_fpavf_pool_set_range(uintptr_t handle,unsigned long memsz,void * memva,uint16_t gpool)394 octeontx_fpavf_pool_set_range(uintptr_t handle, unsigned long memsz,
395 void *memva, uint16_t gpool)
396 {
397 uint64_t va_end;
398
399 if (unlikely(!handle))
400 return -ENODEV;
401
402 va_end = (uintptr_t)memva + memsz;
403 va_end &= ~RTE_CACHE_LINE_MASK;
404
405 /* VHPOOL setup */
406 fpavf_write64((uintptr_t)memva,
407 (void *)((uintptr_t)handle +
408 FPA_VF_VHPOOL_START_ADDR(gpool)));
409 fpavf_write64(va_end,
410 (void *)((uintptr_t)handle +
411 FPA_VF_VHPOOL_END_ADDR(gpool)));
412 return 0;
413 }
414
415 static int
octeontx_fpapf_start_count(uint16_t gpool_index)416 octeontx_fpapf_start_count(uint16_t gpool_index)
417 {
418 int ret = 0;
419 struct octeontx_mbox_hdr hdr = {0};
420
421 if (gpool_index >= FPA_MAX_POOL) {
422 ret = -EINVAL;
423 goto err;
424 }
425
426 hdr.coproc = FPA_COPROC;
427 hdr.msg = FPA_START_COUNT;
428 hdr.vfid = gpool_index;
429 ret = octeontx_mbox_send(&hdr, NULL, 0, NULL, 0);
430 if (ret < 0) {
431 fpavf_log_err("Could not start buffer counting for ");
432 fpavf_log_err("FPA pool %d. Err=%d. FuncErr=%d\n",
433 gpool_index, ret, hdr.res_code);
434 ret = -EINVAL;
435 goto err;
436 }
437
438 err:
439 return ret;
440 }
441
442 static __rte_always_inline int
octeontx_fpavf_free(unsigned int gpool)443 octeontx_fpavf_free(unsigned int gpool)
444 {
445 struct fpavf_res *res = octeontx_get_fpavf(gpool);
446 int ret = 0;
447
448 if (gpool >= FPA_MAX_POOL) {
449 ret = -EINVAL;
450 goto err;
451 }
452
453 /* Pool is free */
454 if (res != NULL)
455 res->is_inuse = false;
456
457 err:
458 return ret;
459 }
460
461 static __rte_always_inline int
octeontx_gpool_free(uint16_t gpool)462 octeontx_gpool_free(uint16_t gpool)
463 {
464 struct fpavf_res *res = octeontx_get_fpavf(gpool);
465
466 if (res && res->sz128 != 0) {
467 res->sz128 = 0;
468 return 0;
469 }
470 return -EINVAL;
471 }
472
473 /*
474 * Return buffer size for a given pool
475 */
476 int
octeontx_fpa_bufpool_block_size(uintptr_t handle)477 octeontx_fpa_bufpool_block_size(uintptr_t handle)
478 {
479 struct fpavf_res *res = NULL;
480 uint8_t gpool;
481
482 if (unlikely(!octeontx_fpa_handle_valid(handle)))
483 return -EINVAL;
484
485 /* get the gpool */
486 gpool = octeontx_fpa_bufpool_gpool(handle);
487 res = octeontx_get_fpavf(gpool);
488 return res ? FPA_CACHE_LINE_2_OBJSZ(res->sz128) : 0;
489 }
490
491 int
octeontx_fpa_bufpool_free_count(uintptr_t handle)492 octeontx_fpa_bufpool_free_count(uintptr_t handle)
493 {
494 uint64_t cnt, limit, avail;
495 uint8_t gpool;
496 uint16_t gaura;
497 uintptr_t pool_bar;
498
499 if (unlikely(!octeontx_fpa_handle_valid(handle)))
500 return -EINVAL;
501
502 /* get the gpool */
503 gpool = octeontx_fpa_bufpool_gpool(handle);
504 /* get the aura */
505 gaura = octeontx_fpa_bufpool_gaura(handle);
506
507 /* Get pool bar address from handle */
508 pool_bar = handle & ~(uint64_t)FPA_GPOOL_MASK;
509
510 cnt = fpavf_read64((void *)((uintptr_t)pool_bar +
511 FPA_VF_VHAURA_CNT(gaura)));
512 limit = fpavf_read64((void *)((uintptr_t)pool_bar +
513 FPA_VF_VHAURA_CNT_LIMIT(gaura)));
514
515 avail = fpavf_read64((void *)((uintptr_t)pool_bar +
516 FPA_VF_VHPOOL_AVAILABLE(gpool)));
517
518 return RTE_MIN(avail, (limit - cnt));
519 }
520
521 uintptr_t
octeontx_fpa_bufpool_create(unsigned int object_size,unsigned int object_count,unsigned int buf_offset,int node_id)522 octeontx_fpa_bufpool_create(unsigned int object_size, unsigned int object_count,
523 unsigned int buf_offset, int node_id)
524 {
525 unsigned int gpool;
526 unsigned int gaura;
527 uintptr_t gpool_handle;
528 uintptr_t pool_bar;
529 int res;
530
531 RTE_SET_USED(node_id);
532 RTE_BUILD_BUG_ON(sizeof(struct rte_mbuf) > OCTEONTX_FPAVF_BUF_OFFSET);
533
534 octeontx_mbox_init();
535 object_size = RTE_CACHE_LINE_ROUNDUP(object_size);
536 if (object_size > FPA_MAX_OBJ_SIZE) {
537 errno = EINVAL;
538 goto error_end;
539 }
540
541 rte_spinlock_lock(&fpadev.lock);
542 res = octeontx_fpa_gpool_alloc(object_size);
543
544 /* Bail if failed */
545 if (unlikely(res < 0)) {
546 errno = res;
547 goto error_unlock;
548 }
549
550 /* get fpavf */
551 gpool = res;
552
553 /* get pool handle */
554 gpool_handle = octeontx_fpa_gpool2handle(gpool);
555 if (!octeontx_fpa_handle_valid(gpool_handle)) {
556 errno = ENOSPC;
557 goto error_gpool_free;
558 }
559
560 /* Get pool bar address from handle */
561 pool_bar = gpool_handle & ~(uint64_t)FPA_GPOOL_MASK;
562
563 res = octeontx_fpapf_pool_setup(gpool, object_size, buf_offset,
564 object_count);
565 if (res < 0) {
566 errno = res;
567 goto error_gpool_free;
568 }
569
570 /* populate AURA fields */
571 res = octeontx_fpapf_aura_attach(gpool);
572 if (res < 0) {
573 errno = res;
574 goto error_pool_destroy;
575 }
576
577 gaura = FPA_AURA_IDX(gpool);
578
579 /* Release lock */
580 rte_spinlock_unlock(&fpadev.lock);
581
582 /* populate AURA registers */
583 fpavf_write64(object_count, (void *)((uintptr_t)pool_bar +
584 FPA_VF_VHAURA_CNT(gaura)));
585 fpavf_write64(object_count, (void *)((uintptr_t)pool_bar +
586 FPA_VF_VHAURA_CNT_LIMIT(gaura)));
587 fpavf_write64(object_count + 1, (void *)((uintptr_t)pool_bar +
588 FPA_VF_VHAURA_CNT_THRESHOLD(gaura)));
589
590 octeontx_fpapf_start_count(gpool);
591
592 return gpool_handle;
593
594 error_pool_destroy:
595 octeontx_fpavf_free(gpool);
596 octeontx_fpapf_pool_destroy(gpool);
597 error_gpool_free:
598 octeontx_gpool_free(gpool);
599 error_unlock:
600 rte_spinlock_unlock(&fpadev.lock);
601 error_end:
602 return (uintptr_t)NULL;
603 }
604
605 /*
606 * Destroy a buffer pool.
607 */
608 int
octeontx_fpa_bufpool_destroy(uintptr_t handle,int node_id)609 octeontx_fpa_bufpool_destroy(uintptr_t handle, int node_id)
610 {
611 void **node, **curr, *head = NULL;
612 uint64_t sz;
613 uint64_t cnt, avail;
614 uint8_t gpool;
615 uint16_t gaura;
616 uintptr_t pool_bar;
617 int ret;
618
619 RTE_SET_USED(node_id);
620
621 /* Wait for all outstanding writes to be committed */
622 rte_smp_wmb();
623
624 if (unlikely(!octeontx_fpa_handle_valid(handle)))
625 return -EINVAL;
626
627 /* get the pool */
628 gpool = octeontx_fpa_bufpool_gpool(handle);
629 /* get the aura */
630 gaura = octeontx_fpa_bufpool_gaura(handle);
631
632 /* Get pool bar address from handle */
633 pool_bar = handle & ~(uint64_t)FPA_GPOOL_MASK;
634
635 /* Check for no outstanding buffers */
636 cnt = fpavf_read64((void *)((uintptr_t)pool_bar +
637 FPA_VF_VHAURA_CNT(gaura)));
638 if (cnt) {
639 fpavf_log_dbg("buffer exist in pool cnt %" PRId64 "\n", cnt);
640 return -EBUSY;
641 }
642
643 rte_spinlock_lock(&fpadev.lock);
644
645 avail = fpavf_read64((void *)((uintptr_t)pool_bar +
646 FPA_VF_VHPOOL_AVAILABLE(gpool)));
647
648 /* Prepare to empty the entire POOL */
649 fpavf_write64(avail, (void *)((uintptr_t)pool_bar +
650 FPA_VF_VHAURA_CNT_LIMIT(gaura)));
651 fpavf_write64(avail + 1, (void *)((uintptr_t)pool_bar +
652 FPA_VF_VHAURA_CNT_THRESHOLD(gaura)));
653
654 /* Empty the pool */
655 /* Invalidate the POOL */
656 octeontx_gpool_free(gpool);
657
658 /* Process all buffers in the pool */
659 while (avail--) {
660
661 /* Yank a buffer from the pool */
662 node = (void *)(uintptr_t)
663 fpavf_read64((void *)
664 (pool_bar + FPA_VF_VHAURA_OP_ALLOC(gaura)));
665
666 if (node == NULL) {
667 fpavf_log_err("GAURA[%u] missing %" PRIx64 " buf\n",
668 gaura, avail);
669 break;
670 }
671
672 /* Imsert it into an ordered linked list */
673 for (curr = &head; curr[0] != NULL; curr = curr[0]) {
674 if ((uintptr_t)node <= (uintptr_t)curr[0])
675 break;
676 }
677 node[0] = curr[0];
678 curr[0] = node;
679 }
680
681 /* Verify the linked list to be a perfect series */
682 sz = octeontx_fpa_bufpool_block_size(handle) << 7;
683 for (curr = head; curr != NULL && curr[0] != NULL;
684 curr = curr[0]) {
685 if (curr == curr[0] ||
686 ((uintptr_t)curr != ((uintptr_t)curr[0] - sz))) {
687 fpavf_log_err("POOL# %u buf sequence err (%p vs. %p)\n",
688 gpool, curr, curr[0]);
689 }
690 }
691
692 /* Disable pool operation */
693 fpavf_write64(~0ul, (void *)((uintptr_t)pool_bar +
694 FPA_VF_VHPOOL_START_ADDR(gpool)));
695 fpavf_write64(~0ul, (void *)((uintptr_t)pool_bar +
696 FPA_VF_VHPOOL_END_ADDR(gpool)));
697
698 (void)octeontx_fpapf_pool_destroy(gpool);
699
700 /* Deactivate the AURA */
701 fpavf_write64(0, (void *)((uintptr_t)pool_bar +
702 FPA_VF_VHAURA_CNT_LIMIT(gaura)));
703 fpavf_write64(0, (void *)((uintptr_t)pool_bar +
704 FPA_VF_VHAURA_CNT_THRESHOLD(gaura)));
705
706 ret = octeontx_fpapf_aura_detach(gpool);
707 if (ret) {
708 fpavf_log_err("Failed to dettach gaura %u. error code=%d\n",
709 gpool, ret);
710 }
711
712 /* Free VF */
713 (void)octeontx_fpavf_free(gpool);
714
715 rte_spinlock_unlock(&fpadev.lock);
716 return 0;
717 }
718
719 static void
octeontx_fpavf_setup(void)720 octeontx_fpavf_setup(void)
721 {
722 uint8_t i;
723 static bool init_once;
724
725 if (!init_once) {
726 rte_spinlock_init(&fpadev.lock);
727 fpadev.total_gpool_cnt = 0;
728
729 for (i = 0; i < FPA_VF_MAX; i++) {
730
731 fpadev.pool[i].domain_id = ~0;
732 fpadev.pool[i].stack_ln_ptr = 0;
733 fpadev.pool[i].sz128 = 0;
734 fpadev.pool[i].bar0 = NULL;
735 fpadev.pool[i].pool_stack_base = NULL;
736 fpadev.pool[i].is_inuse = false;
737 }
738 init_once = 1;
739 }
740 }
741
742 static int
octeontx_fpavf_identify(void * bar0)743 octeontx_fpavf_identify(void *bar0)
744 {
745 uint64_t val;
746 uint16_t domain_id;
747 uint16_t vf_id;
748 uint64_t stack_ln_ptr;
749 static uint16_t vf_idx;
750
751 val = fpavf_read64((void *)((uintptr_t)bar0 +
752 FPA_VF_VHAURA_CNT_THRESHOLD(0)));
753
754 domain_id = (val >> 8) & 0xffff;
755 vf_id = (val >> 24) & 0xffff;
756
757 stack_ln_ptr = fpavf_read64((void *)((uintptr_t)bar0 +
758 FPA_VF_VHPOOL_THRESHOLD(0)));
759 if (vf_idx >= FPA_VF_MAX) {
760 fpavf_log_err("vf_id(%d) greater than max vf (32)\n", vf_id);
761 return -E2BIG;
762 }
763
764 fpadev.pool[vf_idx].domain_id = domain_id;
765 fpadev.pool[vf_idx].vf_id = vf_id;
766 fpadev.pool[vf_idx].bar0 = bar0;
767 fpadev.pool[vf_idx].stack_ln_ptr = stack_ln_ptr;
768
769 /* SUCCESS */
770 return vf_idx++;
771 }
772
773 /* FPAVF pcie device aka mempool probe */
774 static int
fpavf_probe(struct rte_pci_driver * pci_drv,struct rte_pci_device * pci_dev)775 fpavf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
776 {
777 uint8_t *idreg;
778 int res;
779 struct fpavf_res *fpa = NULL;
780
781 RTE_SET_USED(pci_drv);
782 RTE_SET_USED(fpa);
783
784 /* For secondary processes, the primary has done all the work */
785 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
786 return 0;
787
788 if (pci_dev->mem_resource[0].addr == NULL) {
789 fpavf_log_err("Empty bars %p ", pci_dev->mem_resource[0].addr);
790 return -ENODEV;
791 }
792 idreg = pci_dev->mem_resource[0].addr;
793
794 octeontx_fpavf_setup();
795
796 res = octeontx_fpavf_identify(idreg);
797 if (res < 0)
798 return -1;
799
800 fpa = &fpadev.pool[res];
801 fpadev.total_gpool_cnt++;
802 rte_wmb();
803
804 fpavf_log_dbg("total_fpavfs %d bar0 %p domain %d vf %d stk_ln_ptr 0x%x",
805 fpadev.total_gpool_cnt, fpa->bar0, fpa->domain_id,
806 fpa->vf_id, (unsigned int)fpa->stack_ln_ptr);
807
808 return 0;
809 }
810
811 static const struct rte_pci_id pci_fpavf_map[] = {
812 {
813 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
814 PCI_DEVICE_ID_OCTEONTX_FPA_VF)
815 },
816 {
817 .vendor_id = 0,
818 },
819 };
820
821 static struct rte_pci_driver pci_fpavf = {
822 .id_table = pci_fpavf_map,
823 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
824 .probe = fpavf_probe,
825 };
826
827 RTE_PMD_REGISTER_PCI(octeontx_fpavf, pci_fpavf);
828