1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <math.h> 6 7 #include "roc_api.h" 8 #include "roc_priv.h" 9 10 static inline uint32_t 11 nix_qsize_to_val(enum nix_q_size qsize) 12 { 13 return (16UL << (qsize * 2)); 14 } 15 16 static inline enum nix_q_size 17 nix_qsize_clampup(uint32_t val) 18 { 19 int i = nix_q_size_16; 20 21 for (; i < nix_q_size_max; i++) 22 if (val <= nix_qsize_to_val(i)) 23 break; 24 25 if (i >= nix_q_size_max) 26 i = nix_q_size_max - 1; 27 28 return i; 29 } 30 31 void 32 nix_rq_vwqe_flush(struct roc_nix_rq *rq, uint16_t vwqe_interval) 33 { 34 uint64_t wait_ns; 35 36 if (!roc_model_is_cn10k()) 37 return; 38 /* Due to HW errata writes to VWQE_FLUSH might hang, so instead 39 * wait for max vwqe timeout interval. 40 */ 41 if (rq->vwqe_ena) { 42 wait_ns = rq->vwqe_wait_tmo * (vwqe_interval + 1) * 100; 43 plt_delay_us((wait_ns / 1E3) + 1); 44 } 45 } 46 47 int 48 nix_rq_ena_dis(struct dev *dev, struct roc_nix_rq *rq, bool enable) 49 { 50 struct mbox *mbox = dev->mbox; 51 52 /* Pkts will be dropped silently if RQ is disabled */ 53 if (roc_model_is_cn9k()) { 54 struct nix_aq_enq_req *aq; 55 56 aq = mbox_alloc_msg_nix_aq_enq(mbox); 57 aq->qidx = rq->qid; 58 aq->ctype = NIX_AQ_CTYPE_RQ; 59 aq->op = NIX_AQ_INSTOP_WRITE; 60 61 aq->rq.ena = enable; 62 aq->rq_mask.ena = ~(aq->rq_mask.ena); 63 } else { 64 struct nix_cn10k_aq_enq_req *aq; 65 66 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 67 aq->qidx = rq->qid; 68 aq->ctype = NIX_AQ_CTYPE_RQ; 69 aq->op = NIX_AQ_INSTOP_WRITE; 70 71 aq->rq.ena = enable; 72 aq->rq_mask.ena = ~(aq->rq_mask.ena); 73 } 74 75 return mbox_process(mbox); 76 } 77 78 int 79 roc_nix_rq_ena_dis(struct roc_nix_rq *rq, bool enable) 80 { 81 struct nix *nix = roc_nix_to_nix_priv(rq->roc_nix); 82 int rc; 83 84 rc = nix_rq_ena_dis(&nix->dev, rq, enable); 85 nix_rq_vwqe_flush(rq, nix->vwqe_interval); 86 87 return rc; 88 } 89 90 int 91 nix_rq_cn9k_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, 92 bool cfg, bool ena) 93 { 94 struct mbox *mbox = dev->mbox; 95 struct nix_aq_enq_req *aq; 96 97 aq = mbox_alloc_msg_nix_aq_enq(mbox); 98 aq->qidx = rq->qid; 99 aq->ctype = NIX_AQ_CTYPE_RQ; 100 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 101 102 if (rq->sso_ena) { 103 /* SSO mode */ 104 aq->rq.sso_ena = 1; 105 aq->rq.sso_tt = rq->tt; 106 aq->rq.sso_grp = rq->hwgrp; 107 aq->rq.ena_wqwd = 1; 108 aq->rq.wqe_skip = rq->wqe_skip; 109 aq->rq.wqe_caching = 1; 110 111 aq->rq.good_utag = rq->tag_mask >> 24; 112 aq->rq.bad_utag = rq->tag_mask >> 24; 113 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 114 } else { 115 /* CQ mode */ 116 aq->rq.sso_ena = 0; 117 aq->rq.good_utag = rq->tag_mask >> 24; 118 aq->rq.bad_utag = rq->tag_mask >> 24; 119 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 120 aq->rq.cq = rq->qid; 121 } 122 123 if (rq->ipsech_ena) 124 aq->rq.ipsech_ena = 1; 125 126 aq->rq.spb_ena = 0; 127 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 128 129 /* Sizes must be aligned to 8 bytes */ 130 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 131 return -EINVAL; 132 133 /* Expressed in number of dwords */ 134 aq->rq.first_skip = rq->first_skip / 8; 135 aq->rq.later_skip = rq->later_skip / 8; 136 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 137 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 138 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 139 aq->rq.ena = ena; 140 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 141 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 142 aq->rq.rq_int_ena = 0; 143 /* Many to one reduction */ 144 aq->rq.qint_idx = rq->qid % qints; 145 aq->rq.xqe_drop_ena = 1; 146 147 /* If RED enabled, then fill enable for all cases */ 148 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 149 aq->rq.spb_pool_pass = rq->spb_red_pass; 150 aq->rq.lpb_pool_pass = rq->red_pass; 151 152 aq->rq.spb_pool_drop = rq->spb_red_drop; 153 aq->rq.lpb_pool_drop = rq->red_drop; 154 } 155 156 if (cfg) { 157 if (rq->sso_ena) { 158 /* SSO mode */ 159 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 160 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 161 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 162 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 163 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 164 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 165 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 166 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 167 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 168 } else { 169 /* CQ mode */ 170 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 171 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 172 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 173 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 174 aq->rq_mask.cq = ~aq->rq_mask.cq; 175 } 176 177 if (rq->ipsech_ena) 178 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 179 180 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 181 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 182 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 183 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 184 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 185 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 186 aq->rq_mask.ena = ~aq->rq_mask.ena; 187 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 188 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 189 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 190 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 191 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 192 193 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 194 aq->rq_mask.spb_pool_pass = ~aq->rq_mask.spb_pool_pass; 195 aq->rq_mask.lpb_pool_pass = ~aq->rq_mask.lpb_pool_pass; 196 197 aq->rq_mask.spb_pool_drop = ~aq->rq_mask.spb_pool_drop; 198 aq->rq_mask.lpb_pool_drop = ~aq->rq_mask.lpb_pool_drop; 199 } 200 } 201 202 return 0; 203 } 204 205 int 206 nix_rq_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, bool cfg, 207 bool ena) 208 { 209 struct nix_cn10k_aq_enq_req *aq; 210 struct mbox *mbox = dev->mbox; 211 212 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 213 aq->qidx = rq->qid; 214 aq->ctype = NIX_AQ_CTYPE_RQ; 215 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 216 217 if (rq->sso_ena) { 218 /* SSO mode */ 219 aq->rq.sso_ena = 1; 220 aq->rq.sso_tt = rq->tt; 221 aq->rq.sso_grp = rq->hwgrp; 222 aq->rq.ena_wqwd = 1; 223 aq->rq.wqe_skip = rq->wqe_skip; 224 aq->rq.wqe_caching = 1; 225 226 aq->rq.good_utag = rq->tag_mask >> 24; 227 aq->rq.bad_utag = rq->tag_mask >> 24; 228 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 229 230 if (rq->vwqe_ena) { 231 aq->rq.vwqe_ena = true; 232 aq->rq.vwqe_skip = rq->vwqe_first_skip; 233 /* Maximal Vector size is (2^(MAX_VSIZE_EXP+2)) */ 234 aq->rq.max_vsize_exp = rq->vwqe_max_sz_exp - 2; 235 aq->rq.vtime_wait = rq->vwqe_wait_tmo; 236 aq->rq.wqe_aura = rq->vwqe_aura_handle; 237 } 238 } else { 239 /* CQ mode */ 240 aq->rq.sso_ena = 0; 241 aq->rq.good_utag = rq->tag_mask >> 24; 242 aq->rq.bad_utag = rq->tag_mask >> 24; 243 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 244 aq->rq.cq = rq->qid; 245 } 246 247 if (rq->ipsech_ena) { 248 aq->rq.ipsech_ena = 1; 249 aq->rq.ipsecd_drop_en = 1; 250 } 251 252 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 253 254 /* Sizes must be aligned to 8 bytes */ 255 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 256 return -EINVAL; 257 258 /* Expressed in number of dwords */ 259 aq->rq.first_skip = rq->first_skip / 8; 260 aq->rq.later_skip = rq->later_skip / 8; 261 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 262 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 263 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 264 aq->rq.ena = ena; 265 266 if (rq->spb_ena) { 267 uint32_t spb_sizem1; 268 269 aq->rq.spb_ena = 1; 270 aq->rq.spb_aura = 271 roc_npa_aura_handle_to_aura(rq->spb_aura_handle); 272 273 if (rq->spb_size & 0x7 || 274 rq->spb_size > NIX_RQ_CN10K_SPB_MAX_SIZE) 275 return -EINVAL; 276 277 spb_sizem1 = rq->spb_size / 8; /* Expressed in no. of dwords */ 278 spb_sizem1 -= 1; /* Expressed in size minus one */ 279 aq->rq.spb_sizem1 = spb_sizem1 & 0x3F; 280 aq->rq.spb_high_sizem1 = (spb_sizem1 >> 6) & 0x7; 281 } else { 282 aq->rq.spb_ena = 0; 283 } 284 285 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 286 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 287 aq->rq.rq_int_ena = 0; 288 /* Many to one reduction */ 289 aq->rq.qint_idx = rq->qid % qints; 290 aq->rq.xqe_drop_ena = 1; 291 292 /* If RED enabled, then fill enable for all cases */ 293 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 294 aq->rq.spb_pool_pass = rq->spb_red_pass; 295 aq->rq.lpb_pool_pass = rq->red_pass; 296 aq->rq.wqe_pool_pass = rq->red_pass; 297 aq->rq.xqe_pass = rq->red_pass; 298 299 aq->rq.spb_pool_drop = rq->spb_red_drop; 300 aq->rq.lpb_pool_drop = rq->red_drop; 301 aq->rq.wqe_pool_drop = rq->red_drop; 302 aq->rq.xqe_drop = rq->red_drop; 303 } 304 305 if (cfg) { 306 if (rq->sso_ena) { 307 /* SSO mode */ 308 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 309 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 310 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 311 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 312 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 313 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 314 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 315 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 316 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 317 if (rq->vwqe_ena) { 318 aq->rq_mask.vwqe_ena = ~aq->rq_mask.vwqe_ena; 319 aq->rq_mask.vwqe_skip = ~aq->rq_mask.vwqe_skip; 320 aq->rq_mask.max_vsize_exp = 321 ~aq->rq_mask.max_vsize_exp; 322 aq->rq_mask.vtime_wait = 323 ~aq->rq_mask.vtime_wait; 324 aq->rq_mask.wqe_aura = ~aq->rq_mask.wqe_aura; 325 } 326 } else { 327 /* CQ mode */ 328 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 329 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 330 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 331 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 332 aq->rq_mask.cq = ~aq->rq_mask.cq; 333 } 334 335 if (rq->ipsech_ena) 336 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 337 338 if (rq->spb_ena) { 339 aq->rq_mask.spb_aura = ~aq->rq_mask.spb_aura; 340 aq->rq_mask.spb_sizem1 = ~aq->rq_mask.spb_sizem1; 341 aq->rq_mask.spb_high_sizem1 = 342 ~aq->rq_mask.spb_high_sizem1; 343 } 344 345 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 346 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 347 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 348 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 349 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 350 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 351 aq->rq_mask.ena = ~aq->rq_mask.ena; 352 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 353 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 354 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 355 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 356 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 357 358 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 359 aq->rq_mask.spb_pool_pass = ~aq->rq_mask.spb_pool_pass; 360 aq->rq_mask.lpb_pool_pass = ~aq->rq_mask.lpb_pool_pass; 361 aq->rq_mask.wqe_pool_pass = ~aq->rq_mask.wqe_pool_pass; 362 aq->rq_mask.xqe_pass = ~aq->rq_mask.xqe_pass; 363 364 aq->rq_mask.spb_pool_drop = ~aq->rq_mask.spb_pool_drop; 365 aq->rq_mask.lpb_pool_drop = ~aq->rq_mask.lpb_pool_drop; 366 aq->rq_mask.wqe_pool_drop = ~aq->rq_mask.wqe_pool_drop; 367 aq->rq_mask.xqe_drop = ~aq->rq_mask.xqe_drop; 368 } 369 } 370 371 return 0; 372 } 373 374 int 375 roc_nix_rq_init(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 376 { 377 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 378 struct mbox *mbox = (&nix->dev)->mbox; 379 bool is_cn9k = roc_model_is_cn9k(); 380 struct dev *dev = &nix->dev; 381 int rc; 382 383 if (roc_nix == NULL || rq == NULL) 384 return NIX_ERR_PARAM; 385 386 if (rq->qid >= nix->nb_rx_queues) 387 return NIX_ERR_QUEUE_INVALID_RANGE; 388 389 rq->roc_nix = roc_nix; 390 391 if (is_cn9k) 392 rc = nix_rq_cn9k_cfg(dev, rq, nix->qints, false, ena); 393 else 394 rc = nix_rq_cfg(dev, rq, nix->qints, false, ena); 395 396 if (rc) 397 return rc; 398 399 rc = mbox_process(mbox); 400 if (rc) 401 return rc; 402 403 return nix_tel_node_add_rq(rq); 404 } 405 406 int 407 roc_nix_rq_modify(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 408 { 409 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 410 struct mbox *mbox = (&nix->dev)->mbox; 411 bool is_cn9k = roc_model_is_cn9k(); 412 struct dev *dev = &nix->dev; 413 int rc; 414 415 if (roc_nix == NULL || rq == NULL) 416 return NIX_ERR_PARAM; 417 418 if (rq->qid >= nix->nb_rx_queues) 419 return NIX_ERR_QUEUE_INVALID_RANGE; 420 421 rq->roc_nix = roc_nix; 422 423 if (is_cn9k) 424 rc = nix_rq_cn9k_cfg(dev, rq, nix->qints, true, ena); 425 else 426 rc = nix_rq_cfg(dev, rq, nix->qints, true, ena); 427 428 if (rc) 429 return rc; 430 431 rc = mbox_process(mbox); 432 if (rc) 433 return rc; 434 435 return nix_tel_node_add_rq(rq); 436 } 437 438 int 439 roc_nix_rq_fini(struct roc_nix_rq *rq) 440 { 441 /* Disabling RQ is sufficient */ 442 return roc_nix_rq_ena_dis(rq, false); 443 } 444 445 int 446 roc_nix_cq_init(struct roc_nix *roc_nix, struct roc_nix_cq *cq) 447 { 448 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 449 struct mbox *mbox = (&nix->dev)->mbox; 450 volatile struct nix_cq_ctx_s *cq_ctx; 451 enum nix_q_size qsize; 452 size_t desc_sz; 453 int rc; 454 455 if (cq == NULL) 456 return NIX_ERR_PARAM; 457 458 if (cq->qid >= nix->nb_rx_queues) 459 return NIX_ERR_QUEUE_INVALID_RANGE; 460 461 qsize = nix_qsize_clampup(cq->nb_desc); 462 cq->nb_desc = nix_qsize_to_val(qsize); 463 cq->qmask = cq->nb_desc - 1; 464 cq->door = nix->base + NIX_LF_CQ_OP_DOOR; 465 cq->status = (int64_t *)(nix->base + NIX_LF_CQ_OP_STATUS); 466 cq->wdata = (uint64_t)cq->qid << 32; 467 cq->roc_nix = roc_nix; 468 469 /* CQE of W16 */ 470 desc_sz = cq->nb_desc * NIX_CQ_ENTRY_SZ; 471 cq->desc_base = plt_zmalloc(desc_sz, NIX_CQ_ALIGN); 472 if (cq->desc_base == NULL) { 473 rc = NIX_ERR_NO_MEM; 474 goto fail; 475 } 476 477 if (roc_model_is_cn9k()) { 478 struct nix_aq_enq_req *aq; 479 480 aq = mbox_alloc_msg_nix_aq_enq(mbox); 481 aq->qidx = cq->qid; 482 aq->ctype = NIX_AQ_CTYPE_CQ; 483 aq->op = NIX_AQ_INSTOP_INIT; 484 cq_ctx = &aq->cq; 485 } else { 486 struct nix_cn10k_aq_enq_req *aq; 487 488 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 489 aq->qidx = cq->qid; 490 aq->ctype = NIX_AQ_CTYPE_CQ; 491 aq->op = NIX_AQ_INSTOP_INIT; 492 cq_ctx = &aq->cq; 493 } 494 495 cq_ctx->ena = 1; 496 cq_ctx->caching = 1; 497 cq_ctx->qsize = qsize; 498 cq_ctx->base = (uint64_t)cq->desc_base; 499 cq_ctx->avg_level = 0xff; 500 cq_ctx->cq_err_int_ena = BIT(NIX_CQERRINT_CQE_FAULT); 501 cq_ctx->cq_err_int_ena |= BIT(NIX_CQERRINT_DOOR_ERR); 502 503 /* Many to one reduction */ 504 cq_ctx->qint_idx = cq->qid % nix->qints; 505 /* Map CQ0 [RQ0] to CINT0 and so on till max 64 irqs */ 506 cq_ctx->cint_idx = cq->qid; 507 508 if (roc_model_is_cn96_a0() || roc_model_is_cn95_a0()) { 509 const float rx_cq_skid = NIX_CQ_FULL_ERRATA_SKID; 510 uint16_t min_rx_drop; 511 512 min_rx_drop = ceil(rx_cq_skid / (float)cq->nb_desc); 513 cq_ctx->drop = min_rx_drop; 514 cq_ctx->drop_ena = 1; 515 cq->drop_thresh = min_rx_drop; 516 } else { 517 cq->drop_thresh = NIX_CQ_THRESH_LEVEL; 518 /* Drop processing or red drop cannot be enabled due to 519 * due to packets coming for second pass from CPT. 520 */ 521 if (!roc_nix_inl_inb_is_enabled(roc_nix)) { 522 cq_ctx->drop = cq->drop_thresh; 523 cq_ctx->drop_ena = 1; 524 } 525 } 526 527 /* TX pause frames enable flow ctrl on RX side */ 528 if (nix->tx_pause) { 529 /* Single BPID is allocated for all rx channels for now */ 530 cq_ctx->bpid = nix->bpid[0]; 531 cq_ctx->bp = cq->drop_thresh; 532 cq_ctx->bp_ena = 1; 533 } 534 535 rc = mbox_process(mbox); 536 if (rc) 537 goto free_mem; 538 539 return nix_tel_node_add_cq(cq); 540 541 free_mem: 542 plt_free(cq->desc_base); 543 fail: 544 return rc; 545 } 546 547 int 548 roc_nix_cq_fini(struct roc_nix_cq *cq) 549 { 550 struct mbox *mbox; 551 struct nix *nix; 552 int rc; 553 554 if (cq == NULL) 555 return NIX_ERR_PARAM; 556 557 nix = roc_nix_to_nix_priv(cq->roc_nix); 558 mbox = (&nix->dev)->mbox; 559 560 /* Disable CQ */ 561 if (roc_model_is_cn9k()) { 562 struct nix_aq_enq_req *aq; 563 564 aq = mbox_alloc_msg_nix_aq_enq(mbox); 565 aq->qidx = cq->qid; 566 aq->ctype = NIX_AQ_CTYPE_CQ; 567 aq->op = NIX_AQ_INSTOP_WRITE; 568 aq->cq.ena = 0; 569 aq->cq.bp_ena = 0; 570 aq->cq_mask.ena = ~aq->cq_mask.ena; 571 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 572 } else { 573 struct nix_cn10k_aq_enq_req *aq; 574 575 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 576 aq->qidx = cq->qid; 577 aq->ctype = NIX_AQ_CTYPE_CQ; 578 aq->op = NIX_AQ_INSTOP_WRITE; 579 aq->cq.ena = 0; 580 aq->cq.bp_ena = 0; 581 aq->cq_mask.ena = ~aq->cq_mask.ena; 582 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 583 } 584 585 rc = mbox_process(mbox); 586 if (rc) 587 return rc; 588 589 plt_free(cq->desc_base); 590 return 0; 591 } 592 593 static int 594 sqb_pool_populate(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 595 { 596 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 597 uint16_t sqes_per_sqb, count, nb_sqb_bufs; 598 struct npa_pool_s pool; 599 struct npa_aura_s aura; 600 uint64_t blk_sz; 601 uint64_t iova; 602 int rc; 603 604 blk_sz = nix->sqb_size; 605 if (sq->max_sqe_sz == roc_nix_maxsqesz_w16) 606 sqes_per_sqb = (blk_sz / 8) / 16; 607 else 608 sqes_per_sqb = (blk_sz / 8) / 8; 609 610 sq->nb_desc = PLT_MAX(256U, sq->nb_desc); 611 nb_sqb_bufs = sq->nb_desc / sqes_per_sqb; 612 nb_sqb_bufs += NIX_SQB_LIST_SPACE; 613 /* Clamp up the SQB count */ 614 nb_sqb_bufs = PLT_MIN(roc_nix->max_sqb_count, 615 (uint16_t)PLT_MAX(NIX_DEF_SQB, nb_sqb_bufs)); 616 617 sq->nb_sqb_bufs = nb_sqb_bufs; 618 sq->sqes_per_sqb_log2 = (uint16_t)plt_log2_u32(sqes_per_sqb); 619 sq->nb_sqb_bufs_adj = 620 nb_sqb_bufs - 621 (PLT_ALIGN_MUL_CEIL(nb_sqb_bufs, sqes_per_sqb) / sqes_per_sqb); 622 sq->nb_sqb_bufs_adj = 623 (sq->nb_sqb_bufs_adj * NIX_SQB_LOWER_THRESH) / 100; 624 625 /* Explicitly set nat_align alone as by default pool is with both 626 * nat_align and buf_offset = 1 which we don't want for SQB. 627 */ 628 memset(&pool, 0, sizeof(struct npa_pool_s)); 629 pool.nat_align = 1; 630 631 memset(&aura, 0, sizeof(aura)); 632 aura.fc_ena = 1; 633 if (roc_model_is_cn9k() || roc_model_is_cn10ka_a0()) 634 aura.fc_stype = 0x0; /* STF */ 635 else 636 aura.fc_stype = 0x3; /* STSTP */ 637 aura.fc_addr = (uint64_t)sq->fc; 638 aura.fc_hyst_bits = 0; /* Store count on all updates */ 639 rc = roc_npa_pool_create(&sq->aura_handle, blk_sz, NIX_MAX_SQB, &aura, 640 &pool); 641 if (rc) 642 goto fail; 643 644 sq->sqe_mem = plt_zmalloc(blk_sz * NIX_MAX_SQB, blk_sz); 645 if (sq->sqe_mem == NULL) { 646 rc = NIX_ERR_NO_MEM; 647 goto nomem; 648 } 649 650 /* Fill the initial buffers */ 651 iova = (uint64_t)sq->sqe_mem; 652 for (count = 0; count < NIX_MAX_SQB; count++) { 653 roc_npa_aura_op_free(sq->aura_handle, 0, iova); 654 iova += blk_sz; 655 } 656 roc_npa_aura_op_range_set(sq->aura_handle, (uint64_t)sq->sqe_mem, iova); 657 roc_npa_aura_limit_modify(sq->aura_handle, sq->nb_sqb_bufs); 658 sq->aura_sqb_bufs = NIX_MAX_SQB; 659 660 return rc; 661 nomem: 662 roc_npa_pool_destroy(sq->aura_handle); 663 fail: 664 return rc; 665 } 666 667 static void 668 sq_cn9k_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 669 uint16_t smq) 670 { 671 struct mbox *mbox = (&nix->dev)->mbox; 672 struct nix_aq_enq_req *aq; 673 674 aq = mbox_alloc_msg_nix_aq_enq(mbox); 675 aq->qidx = sq->qid; 676 aq->ctype = NIX_AQ_CTYPE_SQ; 677 aq->op = NIX_AQ_INSTOP_INIT; 678 aq->sq.max_sqe_size = sq->max_sqe_sz; 679 680 aq->sq.max_sqe_size = sq->max_sqe_sz; 681 aq->sq.smq = smq; 682 aq->sq.smq_rr_quantum = rr_quantum; 683 aq->sq.default_chan = nix->tx_chan_base; 684 aq->sq.sqe_stype = NIX_STYPE_STF; 685 aq->sq.ena = 1; 686 aq->sq.sso_ena = !!sq->sso_ena; 687 aq->sq.cq_ena = !!sq->cq_ena; 688 aq->sq.cq = sq->cqid; 689 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 690 aq->sq.sqe_stype = NIX_STYPE_STP; 691 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 692 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 693 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 694 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 695 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 696 697 /* Many to one reduction */ 698 aq->sq.qint_idx = sq->qid % nix->qints; 699 } 700 701 static int 702 sq_cn9k_fini(struct nix *nix, struct roc_nix_sq *sq) 703 { 704 struct mbox *mbox = (&nix->dev)->mbox; 705 struct nix_aq_enq_rsp *rsp; 706 struct nix_aq_enq_req *aq; 707 uint16_t sqes_per_sqb; 708 void *sqb_buf; 709 int rc, count; 710 711 aq = mbox_alloc_msg_nix_aq_enq(mbox); 712 aq->qidx = sq->qid; 713 aq->ctype = NIX_AQ_CTYPE_SQ; 714 aq->op = NIX_AQ_INSTOP_READ; 715 rc = mbox_process_msg(mbox, (void *)&rsp); 716 if (rc) 717 return rc; 718 719 /* Check if sq is already cleaned up */ 720 if (!rsp->sq.ena) 721 return 0; 722 723 /* Disable sq */ 724 aq = mbox_alloc_msg_nix_aq_enq(mbox); 725 aq->qidx = sq->qid; 726 aq->ctype = NIX_AQ_CTYPE_SQ; 727 aq->op = NIX_AQ_INSTOP_WRITE; 728 aq->sq_mask.ena = ~aq->sq_mask.ena; 729 aq->sq.ena = 0; 730 rc = mbox_process(mbox); 731 if (rc) 732 return rc; 733 734 /* Read SQ and free sqb's */ 735 aq = mbox_alloc_msg_nix_aq_enq(mbox); 736 aq->qidx = sq->qid; 737 aq->ctype = NIX_AQ_CTYPE_SQ; 738 aq->op = NIX_AQ_INSTOP_READ; 739 rc = mbox_process_msg(mbox, (void *)&rsp); 740 if (rc) 741 return rc; 742 743 if (aq->sq.smq_pend) 744 plt_err("SQ has pending SQE's"); 745 746 count = aq->sq.sqb_count; 747 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 748 /* Free SQB's that are used */ 749 sqb_buf = (void *)rsp->sq.head_sqb; 750 while (count) { 751 void *next_sqb; 752 753 next_sqb = *(void **)((uintptr_t)sqb_buf + 754 (uint32_t)((sqes_per_sqb - 1) * 755 sq->max_sqe_sz)); 756 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 757 sqb_buf = next_sqb; 758 count--; 759 } 760 761 /* Free next to use sqb */ 762 if (rsp->sq.next_sqb) 763 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 764 return 0; 765 } 766 767 static void 768 sq_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 769 uint16_t smq) 770 { 771 struct mbox *mbox = (&nix->dev)->mbox; 772 struct nix_cn10k_aq_enq_req *aq; 773 774 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 775 aq->qidx = sq->qid; 776 aq->ctype = NIX_AQ_CTYPE_SQ; 777 aq->op = NIX_AQ_INSTOP_INIT; 778 aq->sq.max_sqe_size = sq->max_sqe_sz; 779 780 aq->sq.max_sqe_size = sq->max_sqe_sz; 781 aq->sq.smq = smq; 782 aq->sq.smq_rr_weight = rr_quantum; 783 aq->sq.default_chan = nix->tx_chan_base; 784 aq->sq.sqe_stype = NIX_STYPE_STF; 785 aq->sq.ena = 1; 786 aq->sq.sso_ena = !!sq->sso_ena; 787 aq->sq.cq_ena = !!sq->cq_ena; 788 aq->sq.cq = sq->cqid; 789 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 790 aq->sq.sqe_stype = NIX_STYPE_STP; 791 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 792 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 793 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 794 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 795 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 796 797 /* Many to one reduction */ 798 aq->sq.qint_idx = sq->qid % nix->qints; 799 } 800 801 static int 802 sq_fini(struct nix *nix, struct roc_nix_sq *sq) 803 { 804 struct mbox *mbox = (&nix->dev)->mbox; 805 struct nix_cn10k_aq_enq_rsp *rsp; 806 struct nix_cn10k_aq_enq_req *aq; 807 uint16_t sqes_per_sqb; 808 void *sqb_buf; 809 int rc, count; 810 811 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 812 aq->qidx = sq->qid; 813 aq->ctype = NIX_AQ_CTYPE_SQ; 814 aq->op = NIX_AQ_INSTOP_READ; 815 rc = mbox_process_msg(mbox, (void *)&rsp); 816 if (rc) 817 return rc; 818 819 /* Check if sq is already cleaned up */ 820 if (!rsp->sq.ena) 821 return 0; 822 823 /* Disable sq */ 824 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 825 aq->qidx = sq->qid; 826 aq->ctype = NIX_AQ_CTYPE_SQ; 827 aq->op = NIX_AQ_INSTOP_WRITE; 828 aq->sq_mask.ena = ~aq->sq_mask.ena; 829 aq->sq.ena = 0; 830 rc = mbox_process(mbox); 831 if (rc) 832 return rc; 833 834 /* Read SQ and free sqb's */ 835 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 836 aq->qidx = sq->qid; 837 aq->ctype = NIX_AQ_CTYPE_SQ; 838 aq->op = NIX_AQ_INSTOP_READ; 839 rc = mbox_process_msg(mbox, (void *)&rsp); 840 if (rc) 841 return rc; 842 843 if (aq->sq.smq_pend) 844 plt_err("SQ has pending SQE's"); 845 846 count = aq->sq.sqb_count; 847 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 848 /* Free SQB's that are used */ 849 sqb_buf = (void *)rsp->sq.head_sqb; 850 while (count) { 851 void *next_sqb; 852 853 next_sqb = *(void **)((uintptr_t)sqb_buf + 854 (uint32_t)((sqes_per_sqb - 1) * 855 sq->max_sqe_sz)); 856 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 857 sqb_buf = next_sqb; 858 count--; 859 } 860 861 /* Free next to use sqb */ 862 if (rsp->sq.next_sqb) 863 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 864 return 0; 865 } 866 867 int 868 roc_nix_sq_init(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 869 { 870 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 871 struct mbox *mbox = (&nix->dev)->mbox; 872 uint16_t qid, smq = UINT16_MAX; 873 uint32_t rr_quantum = 0; 874 int rc; 875 876 if (sq == NULL) 877 return NIX_ERR_PARAM; 878 879 qid = sq->qid; 880 if (qid >= nix->nb_tx_queues) 881 return NIX_ERR_QUEUE_INVALID_RANGE; 882 883 sq->roc_nix = roc_nix; 884 /* 885 * Allocate memory for flow control updates from HW. 886 * Alloc one cache line, so that fits all FC_STYPE modes. 887 */ 888 sq->fc = plt_zmalloc(ROC_ALIGN, ROC_ALIGN); 889 if (sq->fc == NULL) { 890 rc = NIX_ERR_NO_MEM; 891 goto fail; 892 } 893 894 rc = sqb_pool_populate(roc_nix, sq); 895 if (rc) 896 goto nomem; 897 898 rc = nix_tm_leaf_data_get(nix, sq->qid, &rr_quantum, &smq); 899 if (rc) { 900 rc = NIX_ERR_TM_LEAF_NODE_GET; 901 goto nomem; 902 } 903 904 /* Init SQ context */ 905 if (roc_model_is_cn9k()) 906 sq_cn9k_init(nix, sq, rr_quantum, smq); 907 else 908 sq_init(nix, sq, rr_quantum, smq); 909 910 rc = mbox_process(mbox); 911 if (rc) 912 goto nomem; 913 914 nix->sqs[qid] = sq; 915 sq->io_addr = nix->base + NIX_LF_OP_SENDX(0); 916 /* Evenly distribute LMT slot for each sq */ 917 if (roc_model_is_cn9k()) { 918 /* Multiple cores/SQ's can use same LMTLINE safely in CN9K */ 919 sq->lmt_addr = (void *)(nix->lmt_base + 920 ((qid & RVU_CN9K_LMT_SLOT_MASK) << 12)); 921 } 922 923 rc = nix_tel_node_add_sq(sq); 924 return rc; 925 nomem: 926 plt_free(sq->fc); 927 fail: 928 return rc; 929 } 930 931 int 932 roc_nix_sq_fini(struct roc_nix_sq *sq) 933 { 934 struct nix *nix; 935 struct mbox *mbox; 936 struct ndc_sync_op *ndc_req; 937 uint16_t qid; 938 int rc = 0; 939 940 if (sq == NULL) 941 return NIX_ERR_PARAM; 942 943 nix = roc_nix_to_nix_priv(sq->roc_nix); 944 mbox = (&nix->dev)->mbox; 945 946 qid = sq->qid; 947 948 rc = nix_tm_sq_flush_pre(sq); 949 950 /* Release SQ context */ 951 if (roc_model_is_cn9k()) 952 rc |= sq_cn9k_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 953 else 954 rc |= sq_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 955 956 /* Sync NDC-NIX-TX for LF */ 957 ndc_req = mbox_alloc_msg_ndc_sync_op(mbox); 958 if (ndc_req == NULL) 959 return -ENOSPC; 960 ndc_req->nix_lf_tx_sync = 1; 961 if (mbox_process(mbox)) 962 rc |= NIX_ERR_NDC_SYNC; 963 964 rc |= nix_tm_sq_flush_post(sq); 965 966 /* Restore limit to max SQB count that the pool was created 967 * for aura drain to succeed. 968 */ 969 roc_npa_aura_limit_modify(sq->aura_handle, NIX_MAX_SQB); 970 rc |= roc_npa_pool_destroy(sq->aura_handle); 971 plt_free(sq->fc); 972 plt_free(sq->sqe_mem); 973 nix->sqs[qid] = NULL; 974 975 return rc; 976 } 977