1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
3 */
4
5 #include "roc_api.h"
6 #include "roc_priv.h"
7
8 uint32_t soft_exp_consumer_cnt;
9
10 PLT_STATIC_ASSERT(ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ ==
11 1UL << ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ_LOG2);
12 PLT_STATIC_ASSERT(ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ == 512);
13 PLT_STATIC_ASSERT(ROC_NIX_INL_ONF_IPSEC_OUTB_SA_SZ ==
14 1UL << ROC_NIX_INL_ONF_IPSEC_OUTB_SA_SZ_LOG2);
15 PLT_STATIC_ASSERT(ROC_NIX_INL_OT_IPSEC_INB_SA_SZ ==
16 1UL << ROC_NIX_INL_OT_IPSEC_INB_SA_SZ_LOG2);
17 PLT_STATIC_ASSERT(ROC_NIX_INL_OT_IPSEC_INB_SA_SZ == 1024);
18 PLT_STATIC_ASSERT(ROC_NIX_INL_OT_IPSEC_OUTB_SA_SZ ==
19 1UL << ROC_NIX_INL_OT_IPSEC_OUTB_SA_SZ_LOG2);
20
21 static int
nix_inl_inb_sa_tbl_setup(struct roc_nix * roc_nix)22 nix_inl_inb_sa_tbl_setup(struct roc_nix *roc_nix)
23 {
24 uint32_t ipsec_in_min_spi = roc_nix->ipsec_in_min_spi;
25 uint32_t ipsec_in_max_spi = roc_nix->ipsec_in_max_spi;
26 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
27 struct roc_nix_ipsec_cfg cfg;
28 uint64_t max_sa, i;
29 size_t inb_sa_sz;
30 void *sa;
31 int rc;
32
33 max_sa = plt_align32pow2(ipsec_in_max_spi - ipsec_in_min_spi + 1);
34
35 /* CN9K SA size is different */
36 if (roc_model_is_cn9k())
37 inb_sa_sz = ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ;
38 else
39 inb_sa_sz = ROC_NIX_INL_OT_IPSEC_INB_SA_SZ;
40
41 /* Alloc contiguous memory for Inbound SA's */
42 nix->inb_sa_sz = inb_sa_sz;
43 nix->inb_spi_mask = max_sa - 1;
44 nix->inb_sa_base = plt_zmalloc(inb_sa_sz * max_sa,
45 ROC_NIX_INL_SA_BASE_ALIGN);
46 if (!nix->inb_sa_base) {
47 plt_err("Failed to allocate memory for Inbound SA");
48 return -ENOMEM;
49 }
50 if (roc_model_is_cn10k()) {
51 for (i = 0; i < max_sa; i++) {
52 sa = ((uint8_t *)nix->inb_sa_base) + (i * inb_sa_sz);
53 roc_ot_ipsec_inb_sa_init(sa, true);
54 }
55 }
56
57 memset(&cfg, 0, sizeof(cfg));
58 cfg.sa_size = inb_sa_sz;
59 cfg.iova = (uintptr_t)nix->inb_sa_base;
60 cfg.max_sa = max_sa;
61 cfg.tt = SSO_TT_ORDERED;
62
63 /* Setup device specific inb SA table */
64 rc = roc_nix_lf_inl_ipsec_cfg(roc_nix, &cfg, true);
65 if (rc) {
66 plt_err("Failed to setup NIX Inbound SA conf, rc=%d", rc);
67 goto free_mem;
68 }
69
70 return 0;
71 free_mem:
72 plt_free(nix->inb_sa_base);
73 nix->inb_sa_base = NULL;
74 return rc;
75 }
76
77 static int
nix_inl_sa_tbl_release(struct roc_nix * roc_nix)78 nix_inl_sa_tbl_release(struct roc_nix *roc_nix)
79 {
80 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
81 int rc;
82
83 rc = roc_nix_lf_inl_ipsec_cfg(roc_nix, NULL, false);
84 if (rc) {
85 plt_err("Failed to disable Inbound inline ipsec, rc=%d", rc);
86 return rc;
87 }
88
89 plt_free(nix->inb_sa_base);
90 nix->inb_sa_base = NULL;
91 return 0;
92 }
93
94 struct roc_cpt_lf *
roc_nix_inl_outb_lf_base_get(struct roc_nix * roc_nix)95 roc_nix_inl_outb_lf_base_get(struct roc_nix *roc_nix)
96 {
97 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
98
99 /* NIX Inline config needs to be done */
100 if (!nix->inl_outb_ena || !nix->cpt_lf_base)
101 return NULL;
102
103 return (struct roc_cpt_lf *)nix->cpt_lf_base;
104 }
105
106 uintptr_t
roc_nix_inl_outb_sa_base_get(struct roc_nix * roc_nix)107 roc_nix_inl_outb_sa_base_get(struct roc_nix *roc_nix)
108 {
109 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
110
111 return (uintptr_t)nix->outb_sa_base;
112 }
113
114 uintptr_t
roc_nix_inl_inb_sa_base_get(struct roc_nix * roc_nix,bool inb_inl_dev)115 roc_nix_inl_inb_sa_base_get(struct roc_nix *roc_nix, bool inb_inl_dev)
116 {
117 struct idev_cfg *idev = idev_get_cfg();
118 struct nix_inl_dev *inl_dev;
119 struct nix *nix = NULL;
120
121 if (idev == NULL)
122 return 0;
123
124 if (!inb_inl_dev && roc_nix == NULL)
125 return -EINVAL;
126
127 if (roc_nix) {
128 nix = roc_nix_to_nix_priv(roc_nix);
129 if (!nix->inl_inb_ena)
130 return 0;
131 }
132
133 if (inb_inl_dev) {
134 inl_dev = idev->nix_inl_dev;
135 /* Return inline dev sa base */
136 if (inl_dev)
137 return (uintptr_t)inl_dev->inb_sa_base;
138 return 0;
139 }
140
141 return (uintptr_t)nix->inb_sa_base;
142 }
143
144 uint32_t
roc_nix_inl_inb_spi_range(struct roc_nix * roc_nix,bool inb_inl_dev,uint32_t * min_spi,uint32_t * max_spi)145 roc_nix_inl_inb_spi_range(struct roc_nix *roc_nix, bool inb_inl_dev,
146 uint32_t *min_spi, uint32_t *max_spi)
147 {
148 struct idev_cfg *idev = idev_get_cfg();
149 uint32_t min = 0, max = 0, mask = 0;
150 struct nix_inl_dev *inl_dev;
151 struct nix *nix = NULL;
152
153 if (idev == NULL)
154 return 0;
155
156 if (!inb_inl_dev && roc_nix == NULL)
157 return -EINVAL;
158
159 inl_dev = idev->nix_inl_dev;
160 if (inb_inl_dev) {
161 if (inl_dev == NULL)
162 goto exit;
163 min = inl_dev->ipsec_in_min_spi;
164 max = inl_dev->ipsec_in_max_spi;
165 mask = inl_dev->inb_spi_mask;
166 } else {
167 nix = roc_nix_to_nix_priv(roc_nix);
168 if (!nix->inl_inb_ena)
169 goto exit;
170 min = roc_nix->ipsec_in_min_spi;
171 max = roc_nix->ipsec_in_max_spi;
172 mask = nix->inb_spi_mask;
173 }
174 exit:
175 if (min_spi)
176 *min_spi = min;
177 if (max_spi)
178 *max_spi = max;
179 return mask;
180 }
181
182 uint32_t
roc_nix_inl_inb_sa_sz(struct roc_nix * roc_nix,bool inl_dev_sa)183 roc_nix_inl_inb_sa_sz(struct roc_nix *roc_nix, bool inl_dev_sa)
184 {
185 struct idev_cfg *idev = idev_get_cfg();
186 struct nix_inl_dev *inl_dev;
187 struct nix *nix;
188
189 if (idev == NULL)
190 return 0;
191
192 if (!inl_dev_sa && roc_nix == NULL)
193 return -EINVAL;
194
195 if (roc_nix) {
196 nix = roc_nix_to_nix_priv(roc_nix);
197 if (!inl_dev_sa)
198 return nix->inb_sa_sz;
199 }
200
201 if (inl_dev_sa) {
202 inl_dev = idev->nix_inl_dev;
203 if (inl_dev)
204 return inl_dev->inb_sa_sz;
205 }
206
207 return 0;
208 }
209
210 uintptr_t
roc_nix_inl_inb_sa_get(struct roc_nix * roc_nix,bool inb_inl_dev,uint32_t spi)211 roc_nix_inl_inb_sa_get(struct roc_nix *roc_nix, bool inb_inl_dev, uint32_t spi)
212 {
213 uint32_t max_spi = 0, min_spi = 0, mask;
214 uintptr_t sa_base;
215 uint64_t sz;
216
217 sa_base = roc_nix_inl_inb_sa_base_get(roc_nix, inb_inl_dev);
218 /* Check if SA base exists */
219 if (!sa_base)
220 return 0;
221
222 /* Get SA size */
223 sz = roc_nix_inl_inb_sa_sz(roc_nix, inb_inl_dev);
224 if (!sz)
225 return 0;
226
227 if (roc_nix && roc_nix->custom_sa_action)
228 return (sa_base + (spi * sz));
229
230 /* Check if SPI is in range */
231 mask = roc_nix_inl_inb_spi_range(roc_nix, inb_inl_dev, &min_spi,
232 &max_spi);
233 if (spi > max_spi || spi < min_spi)
234 plt_nix_dbg("Inbound SA SPI %u not in range (%u..%u)", spi,
235 min_spi, max_spi);
236
237 /* Basic logic of SPI->SA for now */
238 return (sa_base + ((spi & mask) * sz));
239 }
240
241 int
roc_nix_reassembly_configure(uint32_t max_wait_time,uint16_t max_frags)242 roc_nix_reassembly_configure(uint32_t max_wait_time, uint16_t max_frags)
243 {
244 struct idev_cfg *idev = idev_get_cfg();
245 struct roc_cpt *roc_cpt;
246 struct roc_cpt_rxc_time_cfg cfg;
247
248 PLT_SET_USED(max_frags);
249 roc_cpt = idev->cpt;
250 if (!roc_cpt) {
251 plt_err("Cannot support inline inbound, cryptodev not probed");
252 return -ENOTSUP;
253 }
254
255 cfg.step = (max_wait_time * 1000 / ROC_NIX_INL_REAS_ACTIVE_LIMIT);
256 cfg.zombie_limit = ROC_NIX_INL_REAS_ZOMBIE_LIMIT;
257 cfg.zombie_thres = ROC_NIX_INL_REAS_ZOMBIE_THRESHOLD;
258 cfg.active_limit = ROC_NIX_INL_REAS_ACTIVE_LIMIT;
259 cfg.active_thres = ROC_NIX_INL_REAS_ACTIVE_THRESHOLD;
260
261 return roc_cpt_rxc_time_cfg(roc_cpt, &cfg);
262 }
263
264 int
roc_nix_inl_inb_init(struct roc_nix * roc_nix)265 roc_nix_inl_inb_init(struct roc_nix *roc_nix)
266 {
267 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
268 struct idev_cfg *idev = idev_get_cfg();
269 struct roc_cpt *roc_cpt;
270 uint16_t param1;
271 int rc;
272
273 if (idev == NULL)
274 return -ENOTSUP;
275
276 /* Unless we have another mechanism to trigger
277 * onetime Inline config in CPTPF, we cannot
278 * support without CPT being probed.
279 */
280 roc_cpt = idev->cpt;
281 if (!roc_cpt) {
282 plt_err("Cannot support inline inbound, cryptodev not probed");
283 return -ENOTSUP;
284 }
285
286 if (roc_model_is_cn9k()) {
287 param1 = ROC_ONF_IPSEC_INB_MAX_L2_SZ;
288 } else {
289 union roc_ot_ipsec_inb_param1 u;
290
291 u.u16 = 0;
292 u.s.esp_trailer_disable = 1;
293 param1 = u.u16;
294 }
295
296 /* Do onetime Inbound Inline config in CPTPF */
297 rc = roc_cpt_inline_ipsec_inb_cfg(roc_cpt, param1, 0);
298 if (rc && rc != -EEXIST) {
299 plt_err("Failed to setup inbound lf, rc=%d", rc);
300 return rc;
301 }
302
303 /* Setup Inbound SA table */
304 rc = nix_inl_inb_sa_tbl_setup(roc_nix);
305 if (rc)
306 return rc;
307
308 nix->inl_inb_ena = true;
309 return 0;
310 }
311
312 int
roc_nix_inl_inb_fini(struct roc_nix * roc_nix)313 roc_nix_inl_inb_fini(struct roc_nix *roc_nix)
314 {
315 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
316
317 if (!nix->inl_inb_ena)
318 return 0;
319
320 nix->inl_inb_ena = false;
321
322 /* Flush Inbound CTX cache entries */
323 roc_nix_cpt_ctx_cache_sync(roc_nix);
324
325 /* Disable Inbound SA */
326 return nix_inl_sa_tbl_release(roc_nix);
327 }
328
329 int
roc_nix_inl_outb_init(struct roc_nix * roc_nix)330 roc_nix_inl_outb_init(struct roc_nix *roc_nix)
331 {
332 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
333 struct idev_cfg *idev = idev_get_cfg();
334 struct roc_cpt_lf *lf_base, *lf;
335 struct dev *dev = &nix->dev;
336 struct msix_offset_rsp *rsp;
337 struct nix_inl_dev *inl_dev;
338 size_t sa_sz, ring_sz;
339 uint16_t sso_pffunc;
340 uint8_t eng_grpmask;
341 uint64_t blkaddr, i;
342 uint64_t *ring_base;
343 uint16_t nb_lf;
344 void *sa_base;
345 int j, rc;
346 void *sa;
347
348 if (idev == NULL)
349 return -ENOTSUP;
350
351 nb_lf = roc_nix->outb_nb_crypto_qs;
352 blkaddr = nix->is_nix1 ? RVU_BLOCK_ADDR_CPT1 : RVU_BLOCK_ADDR_CPT0;
353
354 /* Retrieve inline device if present */
355 inl_dev = idev->nix_inl_dev;
356 sso_pffunc = inl_dev ? inl_dev->dev.pf_func : idev_sso_pffunc_get();
357 /* Use sso_pffunc if explicitly requested */
358 if (roc_nix->ipsec_out_sso_pffunc)
359 sso_pffunc = idev_sso_pffunc_get();
360
361 if (!sso_pffunc) {
362 plt_err("Failed to setup inline outb, need either "
363 "inline device or sso device");
364 return -ENOTSUP;
365 }
366
367 /* Attach CPT LF for outbound */
368 rc = cpt_lfs_attach(dev, blkaddr, true, nb_lf);
369 if (rc) {
370 plt_err("Failed to attach CPT LF for inline outb, rc=%d", rc);
371 return rc;
372 }
373
374 /* Alloc CPT LF */
375 eng_grpmask = (1ULL << ROC_CPT_DFLT_ENG_GRP_SE |
376 1ULL << ROC_CPT_DFLT_ENG_GRP_SE_IE |
377 1ULL << ROC_CPT_DFLT_ENG_GRP_AE);
378 rc = cpt_lfs_alloc(dev, eng_grpmask, blkaddr,
379 !roc_nix->ipsec_out_sso_pffunc);
380 if (rc) {
381 plt_err("Failed to alloc CPT LF resources, rc=%d", rc);
382 goto lf_detach;
383 }
384
385 /* Get msix offsets */
386 rc = cpt_get_msix_offset(dev, &rsp);
387 if (rc) {
388 plt_err("Failed to get CPT LF msix offset, rc=%d", rc);
389 goto lf_free;
390 }
391
392 mbox_memcpy(nix->cpt_msixoff,
393 nix->is_nix1 ? rsp->cpt1_lf_msixoff : rsp->cptlf_msixoff,
394 sizeof(nix->cpt_msixoff));
395
396 /* Alloc required num of cpt lfs */
397 lf_base = plt_zmalloc(nb_lf * sizeof(struct roc_cpt_lf), 0);
398 if (!lf_base) {
399 plt_err("Failed to alloc cpt lf memory");
400 rc = -ENOMEM;
401 goto lf_free;
402 }
403
404 /* Initialize CPT LF's */
405 for (i = 0; i < nb_lf; i++) {
406 lf = &lf_base[i];
407
408 lf->lf_id = i;
409 lf->nb_desc = roc_nix->outb_nb_desc;
410 lf->dev = &nix->dev;
411 lf->msixoff = nix->cpt_msixoff[i];
412 lf->pci_dev = nix->pci_dev;
413
414 /* Setup CPT LF instruction queue */
415 rc = cpt_lf_init(lf);
416 if (rc) {
417 plt_err("Failed to initialize CPT LF, rc=%d", rc);
418 goto lf_fini;
419 }
420
421 /* Associate this CPT LF with NIX PFFUNC */
422 rc = cpt_lf_outb_cfg(dev, sso_pffunc, nix->dev.pf_func, i,
423 true);
424 if (rc) {
425 plt_err("Failed to setup CPT LF->(NIX,SSO) link, rc=%d",
426 rc);
427 goto lf_fini;
428 }
429
430 /* Enable IQ */
431 roc_cpt_iq_enable(lf);
432 }
433
434 if (!roc_nix->ipsec_out_max_sa)
435 goto skip_sa_alloc;
436
437 /* CN9K SA size is different */
438 if (roc_model_is_cn9k())
439 sa_sz = ROC_NIX_INL_ONF_IPSEC_OUTB_SA_SZ;
440 else
441 sa_sz = ROC_NIX_INL_OT_IPSEC_OUTB_SA_SZ;
442 /* Alloc contiguous memory of outbound SA */
443 sa_base = plt_zmalloc(sa_sz * roc_nix->ipsec_out_max_sa,
444 ROC_NIX_INL_SA_BASE_ALIGN);
445 if (!sa_base) {
446 plt_err("Outbound SA base alloc failed");
447 goto lf_fini;
448 }
449 if (roc_model_is_cn10k()) {
450 for (i = 0; i < roc_nix->ipsec_out_max_sa; i++) {
451 sa = ((uint8_t *)sa_base) + (i * sa_sz);
452 roc_ot_ipsec_outb_sa_init(sa);
453 }
454 }
455 nix->outb_sa_base = sa_base;
456 nix->outb_sa_sz = sa_sz;
457
458 skip_sa_alloc:
459
460 nix->cpt_lf_base = lf_base;
461 nix->nb_cpt_lf = nb_lf;
462 nix->outb_err_sso_pffunc = sso_pffunc;
463 nix->inl_outb_ena = true;
464 nix->outb_se_ring_cnt =
465 roc_nix->ipsec_out_max_sa / ROC_IPSEC_ERR_RING_MAX_ENTRY + 1;
466 nix->outb_se_ring_base =
467 roc_nix->port_id * ROC_NIX_SOFT_EXP_PER_PORT_MAX_RINGS;
468
469 if (inl_dev == NULL || !inl_dev->set_soft_exp_poll) {
470 nix->outb_se_ring_cnt = 0;
471 return 0;
472 }
473
474 /* Allocate memory to be used as a ring buffer to poll for
475 * soft expiry event from ucode
476 */
477 ring_sz = (ROC_IPSEC_ERR_RING_MAX_ENTRY + 1) * sizeof(uint64_t);
478 ring_base = inl_dev->sa_soft_exp_ring;
479 for (i = 0; i < nix->outb_se_ring_cnt; i++) {
480 ring_base[nix->outb_se_ring_base + i] =
481 PLT_U64_CAST(plt_zmalloc(ring_sz, 0));
482 if (!ring_base[nix->outb_se_ring_base + i]) {
483 plt_err("Couldn't allocate memory for soft exp ring");
484 while (i--)
485 plt_free(PLT_PTR_CAST(
486 ring_base[nix->outb_se_ring_base + i]));
487 rc = -ENOMEM;
488 goto lf_fini;
489 }
490 }
491
492 return 0;
493
494 lf_fini:
495 for (j = i - 1; j >= 0; j--)
496 cpt_lf_fini(&lf_base[j]);
497 plt_free(lf_base);
498 lf_free:
499 rc |= cpt_lfs_free(dev);
500 lf_detach:
501 rc |= cpt_lfs_detach(dev);
502 return rc;
503 }
504
505 int
roc_nix_inl_outb_fini(struct roc_nix * roc_nix)506 roc_nix_inl_outb_fini(struct roc_nix *roc_nix)
507 {
508 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
509 struct roc_cpt_lf *lf_base = nix->cpt_lf_base;
510 struct idev_cfg *idev = idev_get_cfg();
511 struct dev *dev = &nix->dev;
512 struct nix_inl_dev *inl_dev;
513 uint64_t *ring_base;
514 int i, rc, ret = 0;
515
516 if (!nix->inl_outb_ena)
517 return 0;
518
519 nix->inl_outb_ena = false;
520
521 /* Cleanup CPT LF instruction queue */
522 for (i = 0; i < nix->nb_cpt_lf; i++)
523 cpt_lf_fini(&lf_base[i]);
524
525 /* Free LF resources */
526 rc = cpt_lfs_free(dev);
527 if (rc)
528 plt_err("Failed to free CPT LF resources, rc=%d", rc);
529 ret |= rc;
530
531 /* Detach LF */
532 rc = cpt_lfs_detach(dev);
533 if (rc)
534 plt_err("Failed to detach CPT LF, rc=%d", rc);
535
536 /* Free LF memory */
537 plt_free(lf_base);
538 nix->cpt_lf_base = NULL;
539 nix->nb_cpt_lf = 0;
540
541 /* Free outbound SA base */
542 plt_free(nix->outb_sa_base);
543 nix->outb_sa_base = NULL;
544
545 if (idev && idev->nix_inl_dev && nix->outb_se_ring_cnt) {
546 inl_dev = idev->nix_inl_dev;
547 ring_base = inl_dev->sa_soft_exp_ring;
548 ring_base += nix->outb_se_ring_base;
549
550 for (i = 0; i < nix->outb_se_ring_cnt; i++) {
551 if (ring_base[i])
552 plt_free(PLT_PTR_CAST(ring_base[i]));
553 }
554 }
555
556 ret |= rc;
557 return ret;
558 }
559
560 bool
roc_nix_inl_dev_is_probed(void)561 roc_nix_inl_dev_is_probed(void)
562 {
563 struct idev_cfg *idev = idev_get_cfg();
564
565 if (idev == NULL)
566 return 0;
567
568 return !!idev->nix_inl_dev;
569 }
570
571 bool
roc_nix_inl_inb_is_enabled(struct roc_nix * roc_nix)572 roc_nix_inl_inb_is_enabled(struct roc_nix *roc_nix)
573 {
574 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
575
576 return nix->inl_inb_ena;
577 }
578
579 bool
roc_nix_inl_outb_is_enabled(struct roc_nix * roc_nix)580 roc_nix_inl_outb_is_enabled(struct roc_nix *roc_nix)
581 {
582 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
583
584 return nix->inl_outb_ena;
585 }
586
587 int
roc_nix_inl_dev_rq_get(struct roc_nix_rq * rq)588 roc_nix_inl_dev_rq_get(struct roc_nix_rq *rq)
589 {
590 struct idev_cfg *idev = idev_get_cfg();
591 int port_id = rq->roc_nix->port_id;
592 struct nix_inl_dev *inl_dev;
593 struct roc_nix_rq *inl_rq;
594 uint16_t inl_rq_id;
595 struct dev *dev;
596 int rc;
597
598 if (idev == NULL)
599 return 0;
600
601 inl_dev = idev->nix_inl_dev;
602 /* Nothing to do if no inline device */
603 if (!inl_dev)
604 return 0;
605
606 /* Check if this RQ is already holding reference */
607 if (rq->inl_dev_refs)
608 return 0;
609
610 inl_rq_id = inl_dev->nb_rqs > 1 ? port_id : 0;
611 dev = &inl_dev->dev;
612 inl_rq = &inl_dev->rqs[inl_rq_id];
613
614 /* Just take reference if already inited */
615 if (inl_rq->inl_dev_refs) {
616 inl_rq->inl_dev_refs++;
617 rq->inl_dev_refs = 1;
618 return 0;
619 }
620 memset(inl_rq, 0, sizeof(struct roc_nix_rq));
621
622 /* Take RQ pool attributes from the first ethdev RQ */
623 inl_rq->qid = inl_rq_id;
624 inl_rq->aura_handle = rq->aura_handle;
625 inl_rq->first_skip = rq->first_skip;
626 inl_rq->later_skip = rq->later_skip;
627 inl_rq->lpb_size = rq->lpb_size;
628 inl_rq->lpb_drop_ena = true;
629 inl_rq->spb_ena = rq->spb_ena;
630 inl_rq->spb_aura_handle = rq->spb_aura_handle;
631 inl_rq->spb_size = rq->spb_size;
632 inl_rq->spb_drop_ena = !!rq->spb_ena;
633
634 if (!roc_model_is_cn9k()) {
635 uint64_t aura_limit =
636 roc_npa_aura_op_limit_get(inl_rq->aura_handle);
637 uint64_t aura_shift = plt_log2_u32(aura_limit);
638 uint64_t aura_drop, drop_pc;
639
640 if (aura_shift < 8)
641 aura_shift = 0;
642 else
643 aura_shift = aura_shift - 8;
644
645 /* Set first pass RQ to drop after part of buffers are in
646 * use to avoid metabuf alloc failure. This is needed as long
647 * as we cannot use different aura.
648 */
649 drop_pc = inl_dev->lpb_drop_pc;
650 aura_drop = ((aura_limit * drop_pc) / 100) >> aura_shift;
651 roc_npa_aura_drop_set(inl_rq->aura_handle, aura_drop, true);
652 }
653
654 if (inl_rq->spb_ena) {
655 uint64_t aura_limit =
656 roc_npa_aura_op_limit_get(inl_rq->spb_aura_handle);
657 uint64_t aura_shift = plt_log2_u32(aura_limit);
658 uint64_t aura_drop, drop_pc;
659
660 if (aura_shift < 8)
661 aura_shift = 0;
662 else
663 aura_shift = aura_shift - 8;
664
665 /* Set first pass RQ to drop after part of buffers are in
666 * use to avoid metabuf alloc failure. This is needed as long
667 * as we cannot use different aura.
668 */
669 drop_pc = inl_dev->spb_drop_pc;
670 aura_drop = ((aura_limit * drop_pc) / 100) >> aura_shift;
671 roc_npa_aura_drop_set(inl_rq->spb_aura_handle, aura_drop, true);
672 }
673
674 /* Enable IPSec */
675 inl_rq->ipsech_ena = true;
676
677 inl_rq->flow_tag_width = 20;
678 /* Special tag mask */
679 inl_rq->tag_mask = rq->tag_mask;
680 inl_rq->tt = SSO_TT_ORDERED;
681 inl_rq->hwgrp = 0;
682 inl_rq->wqe_skip = inl_dev->wqe_skip;
683 inl_rq->sso_ena = true;
684
685 /* Prepare and send RQ init mbox */
686 if (roc_model_is_cn9k())
687 rc = nix_rq_cn9k_cfg(dev, inl_rq, inl_dev->qints, false, true);
688 else
689 rc = nix_rq_cfg(dev, inl_rq, inl_dev->qints, false, true);
690 if (rc) {
691 plt_err("Failed to prepare aq_enq msg, rc=%d", rc);
692 return rc;
693 }
694
695 rc = mbox_process(dev->mbox);
696 if (rc) {
697 plt_err("Failed to send aq_enq msg, rc=%d", rc);
698 return rc;
699 }
700
701 inl_rq->inl_dev_refs++;
702 rq->inl_dev_refs = 1;
703 return 0;
704 }
705
706 int
roc_nix_inl_dev_rq_put(struct roc_nix_rq * rq)707 roc_nix_inl_dev_rq_put(struct roc_nix_rq *rq)
708 {
709 struct idev_cfg *idev = idev_get_cfg();
710 int port_id = rq->roc_nix->port_id;
711 struct nix_inl_dev *inl_dev;
712 struct roc_nix_rq *inl_rq;
713 uint16_t inl_rq_id;
714 struct dev *dev;
715 int rc;
716
717 if (idev == NULL)
718 return 0;
719
720 if (!rq->inl_dev_refs)
721 return 0;
722
723 inl_dev = idev->nix_inl_dev;
724 /* Inline device should be there if we have ref */
725 if (!inl_dev) {
726 plt_err("Failed to find inline device with refs");
727 return -EFAULT;
728 }
729
730 dev = &inl_dev->dev;
731 inl_rq_id = inl_dev->nb_rqs > 1 ? port_id : 0;
732 inl_rq = &inl_dev->rqs[inl_rq_id];
733
734 rq->inl_dev_refs = 0;
735 inl_rq->inl_dev_refs--;
736 if (inl_rq->inl_dev_refs)
737 return 0;
738
739 /* There are no more references, disable RQ */
740 rc = nix_rq_ena_dis(dev, inl_rq, false);
741 if (rc)
742 plt_err("Failed to disable inline device rq, rc=%d", rc);
743
744 roc_npa_aura_drop_set(inl_rq->aura_handle, 0, false);
745 if (inl_rq->spb_ena)
746 roc_npa_aura_drop_set(inl_rq->spb_aura_handle, 0, false);
747
748 /* Flush NIX LF for CN10K */
749 nix_rq_vwqe_flush(rq, inl_dev->vwqe_interval);
750
751 return rc;
752 }
753
754 void
roc_nix_inb_mode_set(struct roc_nix * roc_nix,bool use_inl_dev)755 roc_nix_inb_mode_set(struct roc_nix *roc_nix, bool use_inl_dev)
756 {
757 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
758
759 /* Info used by NPC flow rule add */
760 nix->inb_inl_dev = use_inl_dev;
761 }
762
763 int
roc_nix_inl_outb_soft_exp_poll_switch(struct roc_nix * roc_nix,bool poll)764 roc_nix_inl_outb_soft_exp_poll_switch(struct roc_nix *roc_nix, bool poll)
765 {
766 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
767 struct idev_cfg *idev = idev_get_cfg();
768 struct nix_inl_dev *inl_dev;
769 uint16_t ring_idx, i;
770
771 if (!idev || !idev->nix_inl_dev)
772 return 0;
773
774 inl_dev = idev->nix_inl_dev;
775
776 for (i = 0; i < nix->outb_se_ring_cnt; i++) {
777 ring_idx = nix->outb_se_ring_base + i;
778
779 if (poll)
780 plt_bitmap_set(inl_dev->soft_exp_ring_bmap, ring_idx);
781 else
782 plt_bitmap_clear(inl_dev->soft_exp_ring_bmap, ring_idx);
783 }
784
785 if (poll)
786 soft_exp_consumer_cnt++;
787 else
788 soft_exp_consumer_cnt--;
789
790 return 0;
791 }
792
793 bool
roc_nix_inb_is_with_inl_dev(struct roc_nix * roc_nix)794 roc_nix_inb_is_with_inl_dev(struct roc_nix *roc_nix)
795 {
796 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
797
798 return nix->inb_inl_dev;
799 }
800
801 struct roc_nix_rq *
roc_nix_inl_dev_rq(struct roc_nix * roc_nix)802 roc_nix_inl_dev_rq(struct roc_nix *roc_nix)
803 {
804 struct idev_cfg *idev = idev_get_cfg();
805 int port_id = roc_nix->port_id;
806 struct nix_inl_dev *inl_dev;
807 struct roc_nix_rq *inl_rq;
808 uint16_t inl_rq_id;
809
810 if (idev != NULL) {
811 inl_dev = idev->nix_inl_dev;
812 if (inl_dev != NULL) {
813 inl_rq_id = inl_dev->nb_rqs > 1 ? port_id : 0;
814 inl_rq = &inl_dev->rqs[inl_rq_id];
815 if (inl_rq->inl_dev_refs)
816 return inl_rq;
817 }
818 }
819
820 return NULL;
821 }
822
823 uint16_t __roc_api
roc_nix_inl_outb_sso_pffunc_get(struct roc_nix * roc_nix)824 roc_nix_inl_outb_sso_pffunc_get(struct roc_nix *roc_nix)
825 {
826 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
827
828 return nix->outb_err_sso_pffunc;
829 }
830
831 int
roc_nix_inl_cb_register(roc_nix_inl_sso_work_cb_t cb,void * args)832 roc_nix_inl_cb_register(roc_nix_inl_sso_work_cb_t cb, void *args)
833 {
834 struct idev_cfg *idev = idev_get_cfg();
835 struct nix_inl_dev *inl_dev;
836
837 if (idev == NULL)
838 return -EIO;
839
840 inl_dev = idev->nix_inl_dev;
841 if (!inl_dev)
842 return -EIO;
843
844 /* Be silent if registration called with same cb and args */
845 if (inl_dev->work_cb == cb && inl_dev->cb_args == args)
846 return 0;
847
848 /* Don't allow registration again if registered with different cb */
849 if (inl_dev->work_cb)
850 return -EBUSY;
851
852 inl_dev->work_cb = cb;
853 inl_dev->cb_args = args;
854 return 0;
855 }
856
857 int
roc_nix_inl_cb_unregister(roc_nix_inl_sso_work_cb_t cb,void * args)858 roc_nix_inl_cb_unregister(roc_nix_inl_sso_work_cb_t cb, void *args)
859 {
860 struct idev_cfg *idev = idev_get_cfg();
861 struct nix_inl_dev *inl_dev;
862
863 if (idev == NULL)
864 return -ENOENT;
865
866 inl_dev = idev->nix_inl_dev;
867 if (!inl_dev)
868 return -ENOENT;
869
870 if (inl_dev->work_cb != cb || inl_dev->cb_args != args)
871 return -EINVAL;
872
873 inl_dev->work_cb = NULL;
874 inl_dev->cb_args = NULL;
875 return 0;
876 }
877
878 int
roc_nix_inl_inb_tag_update(struct roc_nix * roc_nix,uint32_t tag_const,uint8_t tt)879 roc_nix_inl_inb_tag_update(struct roc_nix *roc_nix, uint32_t tag_const,
880 uint8_t tt)
881 {
882 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
883 struct roc_nix_ipsec_cfg cfg;
884
885 /* Be silent if inline inbound not enabled */
886 if (!nix->inl_inb_ena)
887 return 0;
888
889 memset(&cfg, 0, sizeof(cfg));
890 cfg.sa_size = nix->inb_sa_sz;
891 cfg.iova = (uintptr_t)nix->inb_sa_base;
892 cfg.max_sa = nix->inb_spi_mask + 1;
893 cfg.tt = tt;
894 cfg.tag_const = tag_const;
895
896 return roc_nix_lf_inl_ipsec_cfg(roc_nix, &cfg, true);
897 }
898
899 int
roc_nix_inl_sa_sync(struct roc_nix * roc_nix,void * sa,bool inb,enum roc_nix_inl_sa_sync_op op)900 roc_nix_inl_sa_sync(struct roc_nix *roc_nix, void *sa, bool inb,
901 enum roc_nix_inl_sa_sync_op op)
902 {
903 struct idev_cfg *idev = idev_get_cfg();
904 struct nix_inl_dev *inl_dev = NULL;
905 struct roc_cpt_lf *outb_lf = NULL;
906 union cpt_lf_ctx_reload reload;
907 union cpt_lf_ctx_flush flush;
908 bool get_inl_lf = true;
909 uintptr_t rbase;
910 struct nix *nix;
911
912 /* Nothing much to do on cn9k */
913 if (roc_model_is_cn9k()) {
914 plt_atomic_thread_fence(__ATOMIC_ACQ_REL);
915 return 0;
916 }
917
918 if (idev)
919 inl_dev = idev->nix_inl_dev;
920
921 if (!inl_dev && roc_nix == NULL)
922 return -EINVAL;
923
924 if (roc_nix) {
925 nix = roc_nix_to_nix_priv(roc_nix);
926 outb_lf = nix->cpt_lf_base;
927 if (inb && !nix->inb_inl_dev)
928 get_inl_lf = false;
929 }
930
931 if (inb && get_inl_lf) {
932 outb_lf = NULL;
933 if (inl_dev && inl_dev->attach_cptlf)
934 outb_lf = &inl_dev->cpt_lf;
935 }
936
937 if (outb_lf) {
938 rbase = outb_lf->rbase;
939
940 flush.u = 0;
941 reload.u = 0;
942 switch (op) {
943 case ROC_NIX_INL_SA_OP_FLUSH_INVAL:
944 flush.s.inval = 1;
945 /* fall through */
946 case ROC_NIX_INL_SA_OP_FLUSH:
947 flush.s.cptr = ((uintptr_t)sa) >> 7;
948 plt_write64(flush.u, rbase + CPT_LF_CTX_FLUSH);
949 break;
950 case ROC_NIX_INL_SA_OP_RELOAD:
951 reload.s.cptr = ((uintptr_t)sa) >> 7;
952 plt_write64(reload.u, rbase + CPT_LF_CTX_RELOAD);
953 break;
954 default:
955 return -EINVAL;
956 }
957 return 0;
958 }
959 plt_err("Could not get CPT LF for SA sync");
960 return -ENOTSUP;
961 }
962
963 int
roc_nix_inl_ctx_write(struct roc_nix * roc_nix,void * sa_dptr,void * sa_cptr,bool inb,uint16_t sa_len)964 roc_nix_inl_ctx_write(struct roc_nix *roc_nix, void *sa_dptr, void *sa_cptr,
965 bool inb, uint16_t sa_len)
966 {
967 struct idev_cfg *idev = idev_get_cfg();
968 struct nix_inl_dev *inl_dev = NULL;
969 struct roc_cpt_lf *outb_lf = NULL;
970 union cpt_lf_ctx_flush flush;
971 bool get_inl_lf = true;
972 uintptr_t rbase;
973 struct nix *nix;
974 int rc;
975
976 /* Nothing much to do on cn9k */
977 if (roc_model_is_cn9k()) {
978 plt_atomic_thread_fence(__ATOMIC_ACQ_REL);
979 return 0;
980 }
981
982 if (idev)
983 inl_dev = idev->nix_inl_dev;
984
985 if (!inl_dev && roc_nix == NULL)
986 return -EINVAL;
987
988 if (roc_nix) {
989 nix = roc_nix_to_nix_priv(roc_nix);
990 outb_lf = nix->cpt_lf_base;
991
992 if (inb && !nix->inb_inl_dev)
993 get_inl_lf = false;
994 }
995
996 if (inb && get_inl_lf) {
997 outb_lf = NULL;
998 if (inl_dev && inl_dev->attach_cptlf)
999 outb_lf = &inl_dev->cpt_lf;
1000 }
1001
1002 if (outb_lf) {
1003 rbase = outb_lf->rbase;
1004 flush.u = 0;
1005
1006 rc = roc_cpt_ctx_write(outb_lf, sa_dptr, sa_cptr, sa_len);
1007 if (rc)
1008 return rc;
1009 /* Trigger CTX flush to write dirty data back to DRAM */
1010 flush.s.cptr = ((uintptr_t)sa_cptr) >> 7;
1011 plt_write64(flush.u, rbase + CPT_LF_CTX_FLUSH);
1012
1013 return 0;
1014 }
1015 plt_nix_dbg("Could not get CPT LF for CTX write");
1016 return -ENOTSUP;
1017 }
1018
1019 int
roc_nix_inl_ts_pkind_set(struct roc_nix * roc_nix,bool ts_ena,bool inb_inl_dev)1020 roc_nix_inl_ts_pkind_set(struct roc_nix *roc_nix, bool ts_ena, bool inb_inl_dev)
1021 {
1022 struct idev_cfg *idev = idev_get_cfg();
1023 struct nix_inl_dev *inl_dev = NULL;
1024 void *sa, *sa_base = NULL;
1025 struct nix *nix = NULL;
1026 uint16_t max_spi = 0;
1027 uint32_t rq_refs = 0;
1028 uint8_t pkind = 0;
1029 int i;
1030
1031 if (roc_model_is_cn9k())
1032 return 0;
1033
1034 if (!inb_inl_dev && (roc_nix == NULL))
1035 return -EINVAL;
1036
1037 if (inb_inl_dev) {
1038 if ((idev == NULL) || (idev->nix_inl_dev == NULL))
1039 return 0;
1040 inl_dev = idev->nix_inl_dev;
1041 } else {
1042 nix = roc_nix_to_nix_priv(roc_nix);
1043 if (!nix->inl_inb_ena)
1044 return 0;
1045 sa_base = nix->inb_sa_base;
1046 max_spi = roc_nix->ipsec_in_max_spi;
1047 }
1048
1049 if (inl_dev) {
1050 for (i = 0; i < inl_dev->nb_rqs; i++)
1051 rq_refs += inl_dev->rqs[i].inl_dev_refs;
1052
1053 if (rq_refs == 0) {
1054 inl_dev->ts_ena = ts_ena;
1055 max_spi = inl_dev->ipsec_in_max_spi;
1056 sa_base = inl_dev->inb_sa_base;
1057 } else if (inl_dev->ts_ena != ts_ena) {
1058 if (inl_dev->ts_ena)
1059 plt_err("Inline device is already configured with TS enable");
1060 else
1061 plt_err("Inline device is already configured with TS disable");
1062 return -ENOTSUP;
1063 } else {
1064 return 0;
1065 }
1066 }
1067
1068 pkind = ts_ena ? ROC_IE_OT_CPT_TS_PKIND : ROC_IE_OT_CPT_PKIND;
1069
1070 sa = (uint8_t *)sa_base;
1071 if (pkind == ((struct roc_ot_ipsec_inb_sa *)sa)->w0.s.pkind)
1072 return 0;
1073
1074 for (i = 0; i < max_spi; i++) {
1075 sa = ((uint8_t *)sa_base) +
1076 (i * ROC_NIX_INL_OT_IPSEC_INB_SA_SZ);
1077 ((struct roc_ot_ipsec_inb_sa *)sa)->w0.s.pkind = pkind;
1078 }
1079 return 0;
1080 }
1081
1082 void
roc_nix_inl_dev_lock(void)1083 roc_nix_inl_dev_lock(void)
1084 {
1085 struct idev_cfg *idev = idev_get_cfg();
1086
1087 if (idev != NULL)
1088 plt_spinlock_lock(&idev->nix_inl_dev_lock);
1089 }
1090
1091 void
roc_nix_inl_dev_unlock(void)1092 roc_nix_inl_dev_unlock(void)
1093 {
1094 struct idev_cfg *idev = idev_get_cfg();
1095
1096 if (idev != NULL)
1097 plt_spinlock_unlock(&idev->nix_inl_dev_lock);
1098 }
1099