1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <math.h> 6 7 #include "roc_api.h" 8 #include "roc_priv.h" 9 10 static inline uint32_t 11 nix_qsize_to_val(enum nix_q_size qsize) 12 { 13 return (16UL << (qsize * 2)); 14 } 15 16 static inline enum nix_q_size 17 nix_qsize_clampup(uint32_t val) 18 { 19 int i = nix_q_size_16; 20 21 for (; i < nix_q_size_max; i++) 22 if (val <= nix_qsize_to_val(i)) 23 break; 24 25 if (i >= nix_q_size_max) 26 i = nix_q_size_max - 1; 27 28 return i; 29 } 30 31 void 32 nix_rq_vwqe_flush(struct roc_nix_rq *rq, uint16_t vwqe_interval) 33 { 34 uint64_t wait_ns; 35 36 if (!roc_model_is_cn10k()) 37 return; 38 /* Due to HW errata writes to VWQE_FLUSH might hang, so instead 39 * wait for max vwqe timeout interval. 40 */ 41 if (rq->vwqe_ena) { 42 wait_ns = rq->vwqe_wait_tmo * (vwqe_interval + 1) * 100; 43 plt_delay_us((wait_ns / 1E3) + 1); 44 } 45 } 46 47 int 48 nix_rq_ena_dis(struct dev *dev, struct roc_nix_rq *rq, bool enable) 49 { 50 struct mbox *mbox = dev->mbox; 51 52 /* Pkts will be dropped silently if RQ is disabled */ 53 if (roc_model_is_cn9k()) { 54 struct nix_aq_enq_req *aq; 55 56 aq = mbox_alloc_msg_nix_aq_enq(mbox); 57 if (!aq) 58 return -ENOSPC; 59 60 aq->qidx = rq->qid; 61 aq->ctype = NIX_AQ_CTYPE_RQ; 62 aq->op = NIX_AQ_INSTOP_WRITE; 63 64 aq->rq.ena = enable; 65 aq->rq_mask.ena = ~(aq->rq_mask.ena); 66 } else { 67 struct nix_cn10k_aq_enq_req *aq; 68 69 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 70 if (!aq) 71 return -ENOSPC; 72 73 aq->qidx = rq->qid; 74 aq->ctype = NIX_AQ_CTYPE_RQ; 75 aq->op = NIX_AQ_INSTOP_WRITE; 76 77 aq->rq.ena = enable; 78 aq->rq_mask.ena = ~(aq->rq_mask.ena); 79 } 80 81 return mbox_process(mbox); 82 } 83 84 int 85 roc_nix_rq_ena_dis(struct roc_nix_rq *rq, bool enable) 86 { 87 struct nix *nix = roc_nix_to_nix_priv(rq->roc_nix); 88 int rc; 89 90 rc = nix_rq_ena_dis(&nix->dev, rq, enable); 91 nix_rq_vwqe_flush(rq, nix->vwqe_interval); 92 93 return rc; 94 } 95 96 int 97 nix_rq_cn9k_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, 98 bool cfg, bool ena) 99 { 100 struct mbox *mbox = dev->mbox; 101 struct nix_aq_enq_req *aq; 102 103 aq = mbox_alloc_msg_nix_aq_enq(mbox); 104 if (!aq) 105 return -ENOSPC; 106 107 aq->qidx = rq->qid; 108 aq->ctype = NIX_AQ_CTYPE_RQ; 109 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 110 111 if (rq->sso_ena) { 112 /* SSO mode */ 113 aq->rq.sso_ena = 1; 114 aq->rq.sso_tt = rq->tt; 115 aq->rq.sso_grp = rq->hwgrp; 116 aq->rq.ena_wqwd = 1; 117 aq->rq.wqe_skip = rq->wqe_skip; 118 aq->rq.wqe_caching = 1; 119 120 aq->rq.good_utag = rq->tag_mask >> 24; 121 aq->rq.bad_utag = rq->tag_mask >> 24; 122 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 123 } else { 124 /* CQ mode */ 125 aq->rq.sso_ena = 0; 126 aq->rq.good_utag = rq->tag_mask >> 24; 127 aq->rq.bad_utag = rq->tag_mask >> 24; 128 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 129 aq->rq.cq = rq->qid; 130 } 131 132 if (rq->ipsech_ena) 133 aq->rq.ipsech_ena = 1; 134 135 aq->rq.spb_ena = 0; 136 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 137 138 /* Sizes must be aligned to 8 bytes */ 139 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 140 return -EINVAL; 141 142 /* Expressed in number of dwords */ 143 aq->rq.first_skip = rq->first_skip / 8; 144 aq->rq.later_skip = rq->later_skip / 8; 145 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 146 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 147 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 148 aq->rq.ena = ena; 149 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 150 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 151 aq->rq.rq_int_ena = 0; 152 /* Many to one reduction */ 153 aq->rq.qint_idx = rq->qid % qints; 154 aq->rq.xqe_drop_ena = 1; 155 156 /* If RED enabled, then fill enable for all cases */ 157 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 158 aq->rq.spb_pool_pass = rq->spb_red_pass; 159 aq->rq.lpb_pool_pass = rq->red_pass; 160 161 aq->rq.spb_pool_drop = rq->spb_red_drop; 162 aq->rq.lpb_pool_drop = rq->red_drop; 163 } 164 165 if (cfg) { 166 if (rq->sso_ena) { 167 /* SSO mode */ 168 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 169 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 170 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 171 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 172 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 173 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 174 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 175 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 176 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 177 } else { 178 /* CQ mode */ 179 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 180 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 181 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 182 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 183 aq->rq_mask.cq = ~aq->rq_mask.cq; 184 } 185 186 if (rq->ipsech_ena) 187 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 188 189 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 190 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 191 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 192 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 193 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 194 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 195 aq->rq_mask.ena = ~aq->rq_mask.ena; 196 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 197 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 198 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 199 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 200 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 201 202 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 203 aq->rq_mask.spb_pool_pass = ~aq->rq_mask.spb_pool_pass; 204 aq->rq_mask.lpb_pool_pass = ~aq->rq_mask.lpb_pool_pass; 205 206 aq->rq_mask.spb_pool_drop = ~aq->rq_mask.spb_pool_drop; 207 aq->rq_mask.lpb_pool_drop = ~aq->rq_mask.lpb_pool_drop; 208 } 209 } 210 211 return 0; 212 } 213 214 int 215 nix_rq_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, bool cfg, 216 bool ena) 217 { 218 struct nix_cn10k_aq_enq_req *aq; 219 struct mbox *mbox = dev->mbox; 220 221 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 222 if (!aq) 223 return -ENOSPC; 224 225 aq->qidx = rq->qid; 226 aq->ctype = NIX_AQ_CTYPE_RQ; 227 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 228 229 if (rq->sso_ena) { 230 /* SSO mode */ 231 aq->rq.sso_ena = 1; 232 aq->rq.sso_tt = rq->tt; 233 aq->rq.sso_grp = rq->hwgrp; 234 aq->rq.ena_wqwd = 1; 235 aq->rq.wqe_skip = rq->wqe_skip; 236 aq->rq.wqe_caching = 1; 237 238 aq->rq.good_utag = rq->tag_mask >> 24; 239 aq->rq.bad_utag = rq->tag_mask >> 24; 240 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 241 242 if (rq->vwqe_ena) { 243 aq->rq.vwqe_ena = true; 244 aq->rq.vwqe_skip = rq->vwqe_first_skip; 245 /* Maximal Vector size is (2^(MAX_VSIZE_EXP+2)) */ 246 aq->rq.max_vsize_exp = rq->vwqe_max_sz_exp - 2; 247 aq->rq.vtime_wait = rq->vwqe_wait_tmo; 248 aq->rq.wqe_aura = rq->vwqe_aura_handle; 249 } 250 } else { 251 /* CQ mode */ 252 aq->rq.sso_ena = 0; 253 aq->rq.good_utag = rq->tag_mask >> 24; 254 aq->rq.bad_utag = rq->tag_mask >> 24; 255 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 256 aq->rq.cq = rq->qid; 257 } 258 259 if (rq->ipsech_ena) { 260 aq->rq.ipsech_ena = 1; 261 aq->rq.ipsecd_drop_en = 1; 262 } 263 264 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 265 266 /* Sizes must be aligned to 8 bytes */ 267 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 268 return -EINVAL; 269 270 /* Expressed in number of dwords */ 271 aq->rq.first_skip = rq->first_skip / 8; 272 aq->rq.later_skip = rq->later_skip / 8; 273 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 274 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 275 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 276 aq->rq.ena = ena; 277 278 if (rq->spb_ena) { 279 uint32_t spb_sizem1; 280 281 aq->rq.spb_ena = 1; 282 aq->rq.spb_aura = 283 roc_npa_aura_handle_to_aura(rq->spb_aura_handle); 284 285 if (rq->spb_size & 0x7 || 286 rq->spb_size > NIX_RQ_CN10K_SPB_MAX_SIZE) 287 return -EINVAL; 288 289 spb_sizem1 = rq->spb_size / 8; /* Expressed in no. of dwords */ 290 spb_sizem1 -= 1; /* Expressed in size minus one */ 291 aq->rq.spb_sizem1 = spb_sizem1 & 0x3F; 292 aq->rq.spb_high_sizem1 = (spb_sizem1 >> 6) & 0x7; 293 } else { 294 aq->rq.spb_ena = 0; 295 } 296 297 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 298 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 299 aq->rq.rq_int_ena = 0; 300 /* Many to one reduction */ 301 aq->rq.qint_idx = rq->qid % qints; 302 aq->rq.xqe_drop_ena = 1; 303 304 /* If RED enabled, then fill enable for all cases */ 305 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 306 aq->rq.spb_pool_pass = rq->spb_red_pass; 307 aq->rq.lpb_pool_pass = rq->red_pass; 308 aq->rq.wqe_pool_pass = rq->red_pass; 309 aq->rq.xqe_pass = rq->red_pass; 310 311 aq->rq.spb_pool_drop = rq->spb_red_drop; 312 aq->rq.lpb_pool_drop = rq->red_drop; 313 aq->rq.wqe_pool_drop = rq->red_drop; 314 aq->rq.xqe_drop = rq->red_drop; 315 } 316 317 if (cfg) { 318 if (rq->sso_ena) { 319 /* SSO mode */ 320 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 321 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 322 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 323 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 324 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 325 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 326 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 327 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 328 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 329 if (rq->vwqe_ena) { 330 aq->rq_mask.vwqe_ena = ~aq->rq_mask.vwqe_ena; 331 aq->rq_mask.vwqe_skip = ~aq->rq_mask.vwqe_skip; 332 aq->rq_mask.max_vsize_exp = 333 ~aq->rq_mask.max_vsize_exp; 334 aq->rq_mask.vtime_wait = 335 ~aq->rq_mask.vtime_wait; 336 aq->rq_mask.wqe_aura = ~aq->rq_mask.wqe_aura; 337 } 338 } else { 339 /* CQ mode */ 340 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 341 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 342 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 343 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 344 aq->rq_mask.cq = ~aq->rq_mask.cq; 345 } 346 347 if (rq->ipsech_ena) 348 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 349 350 if (rq->spb_ena) { 351 aq->rq_mask.spb_aura = ~aq->rq_mask.spb_aura; 352 aq->rq_mask.spb_sizem1 = ~aq->rq_mask.spb_sizem1; 353 aq->rq_mask.spb_high_sizem1 = 354 ~aq->rq_mask.spb_high_sizem1; 355 } 356 357 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 358 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 359 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 360 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 361 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 362 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 363 aq->rq_mask.ena = ~aq->rq_mask.ena; 364 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 365 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 366 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 367 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 368 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 369 370 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 371 aq->rq_mask.spb_pool_pass = ~aq->rq_mask.spb_pool_pass; 372 aq->rq_mask.lpb_pool_pass = ~aq->rq_mask.lpb_pool_pass; 373 aq->rq_mask.wqe_pool_pass = ~aq->rq_mask.wqe_pool_pass; 374 aq->rq_mask.xqe_pass = ~aq->rq_mask.xqe_pass; 375 376 aq->rq_mask.spb_pool_drop = ~aq->rq_mask.spb_pool_drop; 377 aq->rq_mask.lpb_pool_drop = ~aq->rq_mask.lpb_pool_drop; 378 aq->rq_mask.wqe_pool_drop = ~aq->rq_mask.wqe_pool_drop; 379 aq->rq_mask.xqe_drop = ~aq->rq_mask.xqe_drop; 380 } 381 } 382 383 return 0; 384 } 385 386 int 387 roc_nix_rq_init(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 388 { 389 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 390 struct mbox *mbox = (&nix->dev)->mbox; 391 bool is_cn9k = roc_model_is_cn9k(); 392 struct dev *dev = &nix->dev; 393 int rc; 394 395 if (roc_nix == NULL || rq == NULL) 396 return NIX_ERR_PARAM; 397 398 if (rq->qid >= nix->nb_rx_queues) 399 return NIX_ERR_QUEUE_INVALID_RANGE; 400 401 rq->roc_nix = roc_nix; 402 403 if (is_cn9k) 404 rc = nix_rq_cn9k_cfg(dev, rq, nix->qints, false, ena); 405 else 406 rc = nix_rq_cfg(dev, rq, nix->qints, false, ena); 407 408 if (rc) 409 return rc; 410 411 rc = mbox_process(mbox); 412 if (rc) 413 return rc; 414 415 return nix_tel_node_add_rq(rq); 416 } 417 418 int 419 roc_nix_rq_modify(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 420 { 421 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 422 struct mbox *mbox = (&nix->dev)->mbox; 423 bool is_cn9k = roc_model_is_cn9k(); 424 struct dev *dev = &nix->dev; 425 int rc; 426 427 if (roc_nix == NULL || rq == NULL) 428 return NIX_ERR_PARAM; 429 430 if (rq->qid >= nix->nb_rx_queues) 431 return NIX_ERR_QUEUE_INVALID_RANGE; 432 433 rq->roc_nix = roc_nix; 434 435 if (is_cn9k) 436 rc = nix_rq_cn9k_cfg(dev, rq, nix->qints, true, ena); 437 else 438 rc = nix_rq_cfg(dev, rq, nix->qints, true, ena); 439 440 if (rc) 441 return rc; 442 443 rc = mbox_process(mbox); 444 if (rc) 445 return rc; 446 447 return nix_tel_node_add_rq(rq); 448 } 449 450 int 451 roc_nix_rq_fini(struct roc_nix_rq *rq) 452 { 453 /* Disabling RQ is sufficient */ 454 return roc_nix_rq_ena_dis(rq, false); 455 } 456 457 int 458 roc_nix_cq_init(struct roc_nix *roc_nix, struct roc_nix_cq *cq) 459 { 460 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 461 struct mbox *mbox = (&nix->dev)->mbox; 462 volatile struct nix_cq_ctx_s *cq_ctx; 463 enum nix_q_size qsize; 464 size_t desc_sz; 465 int rc; 466 467 if (cq == NULL) 468 return NIX_ERR_PARAM; 469 470 if (cq->qid >= nix->nb_rx_queues) 471 return NIX_ERR_QUEUE_INVALID_RANGE; 472 473 qsize = nix_qsize_clampup(cq->nb_desc); 474 cq->nb_desc = nix_qsize_to_val(qsize); 475 cq->qmask = cq->nb_desc - 1; 476 cq->door = nix->base + NIX_LF_CQ_OP_DOOR; 477 cq->status = (int64_t *)(nix->base + NIX_LF_CQ_OP_STATUS); 478 cq->wdata = (uint64_t)cq->qid << 32; 479 cq->roc_nix = roc_nix; 480 481 /* CQE of W16 */ 482 desc_sz = cq->nb_desc * NIX_CQ_ENTRY_SZ; 483 cq->desc_base = plt_zmalloc(desc_sz, NIX_CQ_ALIGN); 484 if (cq->desc_base == NULL) { 485 rc = NIX_ERR_NO_MEM; 486 goto fail; 487 } 488 489 if (roc_model_is_cn9k()) { 490 struct nix_aq_enq_req *aq; 491 492 aq = mbox_alloc_msg_nix_aq_enq(mbox); 493 if (!aq) 494 return -ENOSPC; 495 496 aq->qidx = cq->qid; 497 aq->ctype = NIX_AQ_CTYPE_CQ; 498 aq->op = NIX_AQ_INSTOP_INIT; 499 cq_ctx = &aq->cq; 500 } else { 501 struct nix_cn10k_aq_enq_req *aq; 502 503 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 504 if (!aq) 505 return -ENOSPC; 506 507 aq->qidx = cq->qid; 508 aq->ctype = NIX_AQ_CTYPE_CQ; 509 aq->op = NIX_AQ_INSTOP_INIT; 510 cq_ctx = &aq->cq; 511 } 512 513 cq_ctx->ena = 1; 514 cq_ctx->caching = 1; 515 cq_ctx->qsize = qsize; 516 cq_ctx->base = (uint64_t)cq->desc_base; 517 cq_ctx->avg_level = 0xff; 518 cq_ctx->cq_err_int_ena = BIT(NIX_CQERRINT_CQE_FAULT); 519 cq_ctx->cq_err_int_ena |= BIT(NIX_CQERRINT_DOOR_ERR); 520 521 /* Many to one reduction */ 522 cq_ctx->qint_idx = cq->qid % nix->qints; 523 /* Map CQ0 [RQ0] to CINT0 and so on till max 64 irqs */ 524 cq_ctx->cint_idx = cq->qid; 525 526 if (roc_model_is_cn96_a0() || roc_model_is_cn95_a0()) { 527 const float rx_cq_skid = NIX_CQ_FULL_ERRATA_SKID; 528 uint16_t min_rx_drop; 529 530 min_rx_drop = ceil(rx_cq_skid / (float)cq->nb_desc); 531 cq_ctx->drop = min_rx_drop; 532 cq_ctx->drop_ena = 1; 533 cq->drop_thresh = min_rx_drop; 534 } else { 535 cq->drop_thresh = NIX_CQ_THRESH_LEVEL; 536 /* Drop processing or red drop cannot be enabled due to 537 * due to packets coming for second pass from CPT. 538 */ 539 if (!roc_nix_inl_inb_is_enabled(roc_nix)) { 540 cq_ctx->drop = cq->drop_thresh; 541 cq_ctx->drop_ena = 1; 542 } 543 } 544 545 /* TX pause frames enable flow ctrl on RX side */ 546 if (nix->tx_pause) { 547 /* Single BPID is allocated for all rx channels for now */ 548 cq_ctx->bpid = nix->bpid[0]; 549 cq_ctx->bp = cq->drop_thresh; 550 cq_ctx->bp_ena = 1; 551 } 552 553 rc = mbox_process(mbox); 554 if (rc) 555 goto free_mem; 556 557 return nix_tel_node_add_cq(cq); 558 559 free_mem: 560 plt_free(cq->desc_base); 561 fail: 562 return rc; 563 } 564 565 int 566 roc_nix_cq_fini(struct roc_nix_cq *cq) 567 { 568 struct mbox *mbox; 569 struct nix *nix; 570 int rc; 571 572 if (cq == NULL) 573 return NIX_ERR_PARAM; 574 575 nix = roc_nix_to_nix_priv(cq->roc_nix); 576 mbox = (&nix->dev)->mbox; 577 578 /* Disable CQ */ 579 if (roc_model_is_cn9k()) { 580 struct nix_aq_enq_req *aq; 581 582 aq = mbox_alloc_msg_nix_aq_enq(mbox); 583 if (!aq) 584 return -ENOSPC; 585 586 aq->qidx = cq->qid; 587 aq->ctype = NIX_AQ_CTYPE_CQ; 588 aq->op = NIX_AQ_INSTOP_WRITE; 589 aq->cq.ena = 0; 590 aq->cq.bp_ena = 0; 591 aq->cq_mask.ena = ~aq->cq_mask.ena; 592 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 593 } else { 594 struct nix_cn10k_aq_enq_req *aq; 595 596 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 597 if (!aq) 598 return -ENOSPC; 599 600 aq->qidx = cq->qid; 601 aq->ctype = NIX_AQ_CTYPE_CQ; 602 aq->op = NIX_AQ_INSTOP_WRITE; 603 aq->cq.ena = 0; 604 aq->cq.bp_ena = 0; 605 aq->cq_mask.ena = ~aq->cq_mask.ena; 606 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 607 } 608 609 rc = mbox_process(mbox); 610 if (rc) 611 return rc; 612 613 plt_free(cq->desc_base); 614 return 0; 615 } 616 617 static int 618 sqb_pool_populate(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 619 { 620 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 621 uint16_t sqes_per_sqb, count, nb_sqb_bufs; 622 struct npa_pool_s pool; 623 struct npa_aura_s aura; 624 uint64_t blk_sz; 625 uint64_t iova; 626 int rc; 627 628 blk_sz = nix->sqb_size; 629 if (sq->max_sqe_sz == roc_nix_maxsqesz_w16) 630 sqes_per_sqb = (blk_sz / 8) / 16; 631 else 632 sqes_per_sqb = (blk_sz / 8) / 8; 633 634 sq->nb_desc = PLT_MAX(256U, sq->nb_desc); 635 nb_sqb_bufs = sq->nb_desc / sqes_per_sqb; 636 nb_sqb_bufs += NIX_SQB_LIST_SPACE; 637 /* Clamp up the SQB count */ 638 nb_sqb_bufs = PLT_MIN(roc_nix->max_sqb_count, 639 (uint16_t)PLT_MAX(NIX_DEF_SQB, nb_sqb_bufs)); 640 641 sq->nb_sqb_bufs = nb_sqb_bufs; 642 sq->sqes_per_sqb_log2 = (uint16_t)plt_log2_u32(sqes_per_sqb); 643 sq->nb_sqb_bufs_adj = 644 nb_sqb_bufs - 645 (PLT_ALIGN_MUL_CEIL(nb_sqb_bufs, sqes_per_sqb) / sqes_per_sqb); 646 sq->nb_sqb_bufs_adj = 647 (sq->nb_sqb_bufs_adj * NIX_SQB_LOWER_THRESH) / 100; 648 649 /* Explicitly set nat_align alone as by default pool is with both 650 * nat_align and buf_offset = 1 which we don't want for SQB. 651 */ 652 memset(&pool, 0, sizeof(struct npa_pool_s)); 653 pool.nat_align = 1; 654 655 memset(&aura, 0, sizeof(aura)); 656 aura.fc_ena = 1; 657 if (roc_model_is_cn9k() || roc_model_is_cn10ka_a0()) 658 aura.fc_stype = 0x0; /* STF */ 659 else 660 aura.fc_stype = 0x3; /* STSTP */ 661 aura.fc_addr = (uint64_t)sq->fc; 662 aura.fc_hyst_bits = 0; /* Store count on all updates */ 663 rc = roc_npa_pool_create(&sq->aura_handle, blk_sz, NIX_MAX_SQB, &aura, 664 &pool); 665 if (rc) 666 goto fail; 667 668 sq->sqe_mem = plt_zmalloc(blk_sz * NIX_MAX_SQB, blk_sz); 669 if (sq->sqe_mem == NULL) { 670 rc = NIX_ERR_NO_MEM; 671 goto nomem; 672 } 673 674 /* Fill the initial buffers */ 675 iova = (uint64_t)sq->sqe_mem; 676 for (count = 0; count < NIX_MAX_SQB; count++) { 677 roc_npa_aura_op_free(sq->aura_handle, 0, iova); 678 iova += blk_sz; 679 } 680 681 if (roc_npa_aura_op_available_wait(sq->aura_handle, NIX_MAX_SQB, 0) != 682 NIX_MAX_SQB) { 683 plt_err("Failed to free all pointers to the pool"); 684 rc = NIX_ERR_NO_MEM; 685 goto npa_fail; 686 } 687 688 roc_npa_aura_op_range_set(sq->aura_handle, (uint64_t)sq->sqe_mem, iova); 689 roc_npa_aura_limit_modify(sq->aura_handle, sq->nb_sqb_bufs); 690 sq->aura_sqb_bufs = NIX_MAX_SQB; 691 692 return rc; 693 npa_fail: 694 plt_free(sq->sqe_mem); 695 nomem: 696 roc_npa_pool_destroy(sq->aura_handle); 697 fail: 698 return rc; 699 } 700 701 static int 702 sq_cn9k_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 703 uint16_t smq) 704 { 705 struct mbox *mbox = (&nix->dev)->mbox; 706 struct nix_aq_enq_req *aq; 707 708 aq = mbox_alloc_msg_nix_aq_enq(mbox); 709 if (!aq) 710 return -ENOSPC; 711 712 aq->qidx = sq->qid; 713 aq->ctype = NIX_AQ_CTYPE_SQ; 714 aq->op = NIX_AQ_INSTOP_INIT; 715 aq->sq.max_sqe_size = sq->max_sqe_sz; 716 717 aq->sq.max_sqe_size = sq->max_sqe_sz; 718 aq->sq.smq = smq; 719 aq->sq.smq_rr_quantum = rr_quantum; 720 aq->sq.default_chan = nix->tx_chan_base; 721 aq->sq.sqe_stype = NIX_STYPE_STF; 722 aq->sq.ena = 1; 723 aq->sq.sso_ena = !!sq->sso_ena; 724 aq->sq.cq_ena = !!sq->cq_ena; 725 aq->sq.cq = sq->cqid; 726 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 727 aq->sq.sqe_stype = NIX_STYPE_STP; 728 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 729 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 730 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 731 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 732 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 733 734 /* Many to one reduction */ 735 /* Assigning QINT 0 to all the SQs, an errata exists where NIXTX can 736 * send incorrect QINT_IDX when reporting queue interrupt (QINT). This 737 * might result in software missing the interrupt. 738 */ 739 aq->sq.qint_idx = 0; 740 return 0; 741 } 742 743 static int 744 sq_cn9k_fini(struct nix *nix, struct roc_nix_sq *sq) 745 { 746 struct mbox *mbox = (&nix->dev)->mbox; 747 struct nix_aq_enq_rsp *rsp; 748 struct nix_aq_enq_req *aq; 749 uint16_t sqes_per_sqb; 750 void *sqb_buf; 751 int rc, count; 752 753 aq = mbox_alloc_msg_nix_aq_enq(mbox); 754 if (!aq) 755 return -ENOSPC; 756 757 aq->qidx = sq->qid; 758 aq->ctype = NIX_AQ_CTYPE_SQ; 759 aq->op = NIX_AQ_INSTOP_READ; 760 rc = mbox_process_msg(mbox, (void *)&rsp); 761 if (rc) 762 return rc; 763 764 /* Check if sq is already cleaned up */ 765 if (!rsp->sq.ena) 766 return 0; 767 768 /* Disable sq */ 769 aq = mbox_alloc_msg_nix_aq_enq(mbox); 770 if (!aq) 771 return -ENOSPC; 772 773 aq->qidx = sq->qid; 774 aq->ctype = NIX_AQ_CTYPE_SQ; 775 aq->op = NIX_AQ_INSTOP_WRITE; 776 aq->sq_mask.ena = ~aq->sq_mask.ena; 777 aq->sq.ena = 0; 778 rc = mbox_process(mbox); 779 if (rc) 780 return rc; 781 782 /* Read SQ and free sqb's */ 783 aq = mbox_alloc_msg_nix_aq_enq(mbox); 784 if (!aq) 785 return -ENOSPC; 786 787 aq->qidx = sq->qid; 788 aq->ctype = NIX_AQ_CTYPE_SQ; 789 aq->op = NIX_AQ_INSTOP_READ; 790 rc = mbox_process_msg(mbox, (void *)&rsp); 791 if (rc) 792 return rc; 793 794 if (aq->sq.smq_pend) 795 plt_err("SQ has pending SQE's"); 796 797 count = aq->sq.sqb_count; 798 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 799 /* Free SQB's that are used */ 800 sqb_buf = (void *)rsp->sq.head_sqb; 801 while (count) { 802 void *next_sqb; 803 804 next_sqb = *(void **)((uintptr_t)sqb_buf + 805 (uint32_t)((sqes_per_sqb - 1) * 806 sq->max_sqe_sz)); 807 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 808 sqb_buf = next_sqb; 809 count--; 810 } 811 812 /* Free next to use sqb */ 813 if (rsp->sq.next_sqb) 814 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 815 return 0; 816 } 817 818 static int 819 sq_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 820 uint16_t smq) 821 { 822 struct mbox *mbox = (&nix->dev)->mbox; 823 struct nix_cn10k_aq_enq_req *aq; 824 825 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 826 if (!aq) 827 return -ENOSPC; 828 829 aq->qidx = sq->qid; 830 aq->ctype = NIX_AQ_CTYPE_SQ; 831 aq->op = NIX_AQ_INSTOP_INIT; 832 aq->sq.max_sqe_size = sq->max_sqe_sz; 833 834 aq->sq.max_sqe_size = sq->max_sqe_sz; 835 aq->sq.smq = smq; 836 aq->sq.smq_rr_weight = rr_quantum; 837 aq->sq.default_chan = nix->tx_chan_base; 838 aq->sq.sqe_stype = NIX_STYPE_STF; 839 aq->sq.ena = 1; 840 aq->sq.sso_ena = !!sq->sso_ena; 841 aq->sq.cq_ena = !!sq->cq_ena; 842 aq->sq.cq = sq->cqid; 843 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 844 aq->sq.sqe_stype = NIX_STYPE_STP; 845 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 846 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 847 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 848 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 849 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 850 851 /* Assigning QINT 0 to all the SQs, an errata exists where NIXTX can 852 * send incorrect QINT_IDX when reporting queue interrupt (QINT). This 853 * might result in software missing the interrupt. 854 */ 855 aq->sq.qint_idx = 0; 856 return 0; 857 } 858 859 static int 860 sq_fini(struct nix *nix, struct roc_nix_sq *sq) 861 { 862 struct mbox *mbox = (&nix->dev)->mbox; 863 struct nix_cn10k_aq_enq_rsp *rsp; 864 struct nix_cn10k_aq_enq_req *aq; 865 uint16_t sqes_per_sqb; 866 void *sqb_buf; 867 int rc, count; 868 869 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 870 if (!aq) 871 return -ENOSPC; 872 873 aq->qidx = sq->qid; 874 aq->ctype = NIX_AQ_CTYPE_SQ; 875 aq->op = NIX_AQ_INSTOP_READ; 876 rc = mbox_process_msg(mbox, (void *)&rsp); 877 if (rc) 878 return rc; 879 880 /* Check if sq is already cleaned up */ 881 if (!rsp->sq.ena) 882 return 0; 883 884 /* Disable sq */ 885 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 886 if (!aq) 887 return -ENOSPC; 888 889 aq->qidx = sq->qid; 890 aq->ctype = NIX_AQ_CTYPE_SQ; 891 aq->op = NIX_AQ_INSTOP_WRITE; 892 aq->sq_mask.ena = ~aq->sq_mask.ena; 893 aq->sq.ena = 0; 894 rc = mbox_process(mbox); 895 if (rc) 896 return rc; 897 898 /* Read SQ and free sqb's */ 899 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 900 if (!aq) 901 return -ENOSPC; 902 903 aq->qidx = sq->qid; 904 aq->ctype = NIX_AQ_CTYPE_SQ; 905 aq->op = NIX_AQ_INSTOP_READ; 906 rc = mbox_process_msg(mbox, (void *)&rsp); 907 if (rc) 908 return rc; 909 910 if (aq->sq.smq_pend) 911 plt_err("SQ has pending SQE's"); 912 913 count = aq->sq.sqb_count; 914 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 915 /* Free SQB's that are used */ 916 sqb_buf = (void *)rsp->sq.head_sqb; 917 while (count) { 918 void *next_sqb; 919 920 next_sqb = *(void **)((uintptr_t)sqb_buf + 921 (uint32_t)((sqes_per_sqb - 1) * 922 sq->max_sqe_sz)); 923 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 924 sqb_buf = next_sqb; 925 count--; 926 } 927 928 /* Free next to use sqb */ 929 if (rsp->sq.next_sqb) 930 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 931 return 0; 932 } 933 934 int 935 roc_nix_sq_init(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 936 { 937 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 938 struct mbox *mbox = (&nix->dev)->mbox; 939 uint16_t qid, smq = UINT16_MAX; 940 uint32_t rr_quantum = 0; 941 int rc; 942 943 if (sq == NULL) 944 return NIX_ERR_PARAM; 945 946 qid = sq->qid; 947 if (qid >= nix->nb_tx_queues) 948 return NIX_ERR_QUEUE_INVALID_RANGE; 949 950 sq->roc_nix = roc_nix; 951 /* 952 * Allocate memory for flow control updates from HW. 953 * Alloc one cache line, so that fits all FC_STYPE modes. 954 */ 955 sq->fc = plt_zmalloc(ROC_ALIGN, ROC_ALIGN); 956 if (sq->fc == NULL) { 957 rc = NIX_ERR_NO_MEM; 958 goto fail; 959 } 960 961 rc = sqb_pool_populate(roc_nix, sq); 962 if (rc) 963 goto nomem; 964 965 rc = nix_tm_leaf_data_get(nix, sq->qid, &rr_quantum, &smq); 966 if (rc) { 967 rc = NIX_ERR_TM_LEAF_NODE_GET; 968 goto nomem; 969 } 970 971 /* Init SQ context */ 972 if (roc_model_is_cn9k()) 973 rc = sq_cn9k_init(nix, sq, rr_quantum, smq); 974 else 975 rc = sq_init(nix, sq, rr_quantum, smq); 976 977 if (rc) 978 goto nomem; 979 980 rc = mbox_process(mbox); 981 if (rc) 982 goto nomem; 983 984 nix->sqs[qid] = sq; 985 sq->io_addr = nix->base + NIX_LF_OP_SENDX(0); 986 /* Evenly distribute LMT slot for each sq */ 987 if (roc_model_is_cn9k()) { 988 /* Multiple cores/SQ's can use same LMTLINE safely in CN9K */ 989 sq->lmt_addr = (void *)(nix->lmt_base + 990 ((qid & RVU_CN9K_LMT_SLOT_MASK) << 12)); 991 } 992 993 rc = nix_tel_node_add_sq(sq); 994 return rc; 995 nomem: 996 plt_free(sq->fc); 997 fail: 998 return rc; 999 } 1000 1001 int 1002 roc_nix_sq_fini(struct roc_nix_sq *sq) 1003 { 1004 struct nix *nix; 1005 struct mbox *mbox; 1006 struct ndc_sync_op *ndc_req; 1007 uint16_t qid; 1008 int rc = 0; 1009 1010 if (sq == NULL) 1011 return NIX_ERR_PARAM; 1012 1013 nix = roc_nix_to_nix_priv(sq->roc_nix); 1014 mbox = (&nix->dev)->mbox; 1015 1016 qid = sq->qid; 1017 1018 rc = nix_tm_sq_flush_pre(sq); 1019 1020 /* Release SQ context */ 1021 if (roc_model_is_cn9k()) 1022 rc |= sq_cn9k_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 1023 else 1024 rc |= sq_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 1025 1026 /* Sync NDC-NIX-TX for LF */ 1027 ndc_req = mbox_alloc_msg_ndc_sync_op(mbox); 1028 if (ndc_req == NULL) 1029 return -ENOSPC; 1030 ndc_req->nix_lf_tx_sync = 1; 1031 if (mbox_process(mbox)) 1032 rc |= NIX_ERR_NDC_SYNC; 1033 1034 rc |= nix_tm_sq_flush_post(sq); 1035 1036 /* Restore limit to max SQB count that the pool was created 1037 * for aura drain to succeed. 1038 */ 1039 roc_npa_aura_limit_modify(sq->aura_handle, NIX_MAX_SQB); 1040 rc |= roc_npa_pool_destroy(sq->aura_handle); 1041 plt_free(sq->fc); 1042 plt_free(sq->sqe_mem); 1043 nix->sqs[qid] = NULL; 1044 1045 return rc; 1046 } 1047 1048 void 1049 roc_nix_cq_head_tail_get(struct roc_nix *roc_nix, uint16_t qid, uint32_t *head, 1050 uint32_t *tail) 1051 { 1052 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 1053 uint64_t reg, val; 1054 int64_t *addr; 1055 1056 if (head == NULL || tail == NULL) 1057 return; 1058 1059 reg = (((uint64_t)qid) << 32); 1060 addr = (int64_t *)(nix->base + NIX_LF_CQ_OP_STATUS); 1061 val = roc_atomic64_add_nosync(reg, addr); 1062 if (val & 1063 (BIT_ULL(NIX_CQ_OP_STAT_OP_ERR) | BIT_ULL(NIX_CQ_OP_STAT_CQ_ERR))) 1064 val = 0; 1065 1066 *tail = (uint32_t)(val & 0xFFFFF); 1067 *head = (uint32_t)((val >> 20) & 0xFFFFF); 1068 } 1069 1070 void 1071 roc_nix_sq_head_tail_get(struct roc_nix *roc_nix, uint16_t qid, uint32_t *head, 1072 uint32_t *tail) 1073 { 1074 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 1075 struct roc_nix_sq *sq = nix->sqs[qid]; 1076 uint16_t sqes_per_sqb, sqb_cnt; 1077 uint64_t reg, val; 1078 int64_t *addr; 1079 1080 if (head == NULL || tail == NULL) 1081 return; 1082 1083 reg = (((uint64_t)qid) << 32); 1084 addr = (int64_t *)(nix->base + NIX_LF_SQ_OP_STATUS); 1085 val = roc_atomic64_add_nosync(reg, addr); 1086 if (val & BIT_ULL(NIX_CQ_OP_STAT_OP_ERR)) { 1087 val = 0; 1088 return; 1089 } 1090 1091 *tail = (uint32_t)((val >> 28) & 0x3F); 1092 *head = (uint32_t)((val >> 20) & 0x3F); 1093 sqb_cnt = (uint16_t)(val & 0xFFFF); 1094 1095 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 1096 1097 /* Update tail index as per used sqb count */ 1098 *tail += (sqes_per_sqb * (sqb_cnt - 1)); 1099 } 1100