1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <math.h> 6 7 #include "roc_api.h" 8 #include "roc_priv.h" 9 10 static inline uint32_t 11 nix_qsize_to_val(enum nix_q_size qsize) 12 { 13 return (16UL << (qsize * 2)); 14 } 15 16 static inline enum nix_q_size 17 nix_qsize_clampup(uint32_t val) 18 { 19 int i = nix_q_size_16; 20 21 for (; i < nix_q_size_max; i++) 22 if (val <= nix_qsize_to_val(i)) 23 break; 24 25 if (i >= nix_q_size_max) 26 i = nix_q_size_max - 1; 27 28 return i; 29 } 30 31 int 32 nix_rq_ena_dis(struct dev *dev, struct roc_nix_rq *rq, bool enable) 33 { 34 struct mbox *mbox = dev->mbox; 35 36 /* Pkts will be dropped silently if RQ is disabled */ 37 if (roc_model_is_cn9k()) { 38 struct nix_aq_enq_req *aq; 39 40 aq = mbox_alloc_msg_nix_aq_enq(mbox); 41 aq->qidx = rq->qid; 42 aq->ctype = NIX_AQ_CTYPE_RQ; 43 aq->op = NIX_AQ_INSTOP_WRITE; 44 45 aq->rq.ena = enable; 46 aq->rq_mask.ena = ~(aq->rq_mask.ena); 47 } else { 48 struct nix_cn10k_aq_enq_req *aq; 49 50 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 51 aq->qidx = rq->qid; 52 aq->ctype = NIX_AQ_CTYPE_RQ; 53 aq->op = NIX_AQ_INSTOP_WRITE; 54 55 aq->rq.ena = enable; 56 aq->rq_mask.ena = ~(aq->rq_mask.ena); 57 } 58 59 return mbox_process(mbox); 60 } 61 62 int 63 roc_nix_rq_ena_dis(struct roc_nix_rq *rq, bool enable) 64 { 65 struct nix *nix = roc_nix_to_nix_priv(rq->roc_nix); 66 int rc; 67 68 rc = nix_rq_ena_dis(&nix->dev, rq, enable); 69 70 if (roc_model_is_cn10k()) 71 plt_write64(rq->qid, nix->base + NIX_LF_OP_VWQE_FLUSH); 72 return rc; 73 } 74 75 int 76 nix_rq_cn9k_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, 77 bool cfg, bool ena) 78 { 79 struct mbox *mbox = dev->mbox; 80 struct nix_aq_enq_req *aq; 81 82 aq = mbox_alloc_msg_nix_aq_enq(mbox); 83 aq->qidx = rq->qid; 84 aq->ctype = NIX_AQ_CTYPE_RQ; 85 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 86 87 if (rq->sso_ena) { 88 /* SSO mode */ 89 aq->rq.sso_ena = 1; 90 aq->rq.sso_tt = rq->tt; 91 aq->rq.sso_grp = rq->hwgrp; 92 aq->rq.ena_wqwd = 1; 93 aq->rq.wqe_skip = rq->wqe_skip; 94 aq->rq.wqe_caching = 1; 95 96 aq->rq.good_utag = rq->tag_mask >> 24; 97 aq->rq.bad_utag = rq->tag_mask >> 24; 98 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 99 } else { 100 /* CQ mode */ 101 aq->rq.sso_ena = 0; 102 aq->rq.good_utag = rq->tag_mask >> 24; 103 aq->rq.bad_utag = rq->tag_mask >> 24; 104 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 105 aq->rq.cq = rq->qid; 106 } 107 108 if (rq->ipsech_ena) 109 aq->rq.ipsech_ena = 1; 110 111 aq->rq.spb_ena = 0; 112 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 113 114 /* Sizes must be aligned to 8 bytes */ 115 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 116 return -EINVAL; 117 118 /* Expressed in number of dwords */ 119 aq->rq.first_skip = rq->first_skip / 8; 120 aq->rq.later_skip = rq->later_skip / 8; 121 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 122 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 123 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 124 aq->rq.ena = ena; 125 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 126 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 127 aq->rq.rq_int_ena = 0; 128 /* Many to one reduction */ 129 aq->rq.qint_idx = rq->qid % qints; 130 aq->rq.xqe_drop_ena = 1; 131 132 /* If RED enabled, then fill enable for all cases */ 133 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 134 aq->rq.spb_pool_pass = rq->spb_red_pass; 135 aq->rq.lpb_pool_pass = rq->red_pass; 136 137 aq->rq.spb_pool_drop = rq->spb_red_drop; 138 aq->rq.lpb_pool_drop = rq->red_drop; 139 } 140 141 if (cfg) { 142 if (rq->sso_ena) { 143 /* SSO mode */ 144 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 145 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 146 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 147 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 148 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 149 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 150 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 151 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 152 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 153 } else { 154 /* CQ mode */ 155 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 156 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 157 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 158 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 159 aq->rq_mask.cq = ~aq->rq_mask.cq; 160 } 161 162 if (rq->ipsech_ena) 163 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 164 165 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 166 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 167 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 168 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 169 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 170 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 171 aq->rq_mask.ena = ~aq->rq_mask.ena; 172 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 173 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 174 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 175 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 176 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 177 178 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 179 aq->rq_mask.spb_pool_pass = ~aq->rq_mask.spb_pool_pass; 180 aq->rq_mask.lpb_pool_pass = ~aq->rq_mask.lpb_pool_pass; 181 182 aq->rq_mask.spb_pool_drop = ~aq->rq_mask.spb_pool_drop; 183 aq->rq_mask.lpb_pool_drop = ~aq->rq_mask.lpb_pool_drop; 184 } 185 } 186 187 return 0; 188 } 189 190 int 191 nix_rq_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, bool cfg, 192 bool ena) 193 { 194 struct nix_cn10k_aq_enq_req *aq; 195 struct mbox *mbox = dev->mbox; 196 197 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 198 aq->qidx = rq->qid; 199 aq->ctype = NIX_AQ_CTYPE_RQ; 200 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 201 202 if (rq->sso_ena) { 203 /* SSO mode */ 204 aq->rq.sso_ena = 1; 205 aq->rq.sso_tt = rq->tt; 206 aq->rq.sso_grp = rq->hwgrp; 207 aq->rq.ena_wqwd = 1; 208 aq->rq.wqe_skip = rq->wqe_skip; 209 aq->rq.wqe_caching = 1; 210 211 aq->rq.good_utag = rq->tag_mask >> 24; 212 aq->rq.bad_utag = rq->tag_mask >> 24; 213 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 214 215 if (rq->vwqe_ena) { 216 aq->rq.vwqe_ena = true; 217 aq->rq.vwqe_skip = rq->vwqe_first_skip; 218 /* Maximal Vector size is (2^(MAX_VSIZE_EXP+2)) */ 219 aq->rq.max_vsize_exp = rq->vwqe_max_sz_exp - 2; 220 aq->rq.vtime_wait = rq->vwqe_wait_tmo; 221 aq->rq.wqe_aura = rq->vwqe_aura_handle; 222 } 223 } else { 224 /* CQ mode */ 225 aq->rq.sso_ena = 0; 226 aq->rq.good_utag = rq->tag_mask >> 24; 227 aq->rq.bad_utag = rq->tag_mask >> 24; 228 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 229 aq->rq.cq = rq->qid; 230 } 231 232 if (rq->ipsech_ena) { 233 aq->rq.ipsech_ena = 1; 234 aq->rq.ipsecd_drop_en = 1; 235 } 236 237 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 238 239 /* Sizes must be aligned to 8 bytes */ 240 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 241 return -EINVAL; 242 243 /* Expressed in number of dwords */ 244 aq->rq.first_skip = rq->first_skip / 8; 245 aq->rq.later_skip = rq->later_skip / 8; 246 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 247 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 248 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 249 aq->rq.ena = ena; 250 251 if (rq->spb_ena) { 252 uint32_t spb_sizem1; 253 254 aq->rq.spb_ena = 1; 255 aq->rq.spb_aura = 256 roc_npa_aura_handle_to_aura(rq->spb_aura_handle); 257 258 if (rq->spb_size & 0x7 || 259 rq->spb_size > NIX_RQ_CN10K_SPB_MAX_SIZE) 260 return -EINVAL; 261 262 spb_sizem1 = rq->spb_size / 8; /* Expressed in no. of dwords */ 263 spb_sizem1 -= 1; /* Expressed in size minus one */ 264 aq->rq.spb_sizem1 = spb_sizem1 & 0x3F; 265 aq->rq.spb_high_sizem1 = (spb_sizem1 >> 6) & 0x7; 266 } else { 267 aq->rq.spb_ena = 0; 268 } 269 270 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 271 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 272 aq->rq.rq_int_ena = 0; 273 /* Many to one reduction */ 274 aq->rq.qint_idx = rq->qid % qints; 275 aq->rq.xqe_drop_ena = 1; 276 277 /* If RED enabled, then fill enable for all cases */ 278 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 279 aq->rq.spb_pool_pass = rq->spb_red_pass; 280 aq->rq.lpb_pool_pass = rq->red_pass; 281 aq->rq.wqe_pool_pass = rq->red_pass; 282 aq->rq.xqe_pass = rq->red_pass; 283 284 aq->rq.spb_pool_drop = rq->spb_red_drop; 285 aq->rq.lpb_pool_drop = rq->red_drop; 286 aq->rq.wqe_pool_drop = rq->red_drop; 287 aq->rq.xqe_drop = rq->red_drop; 288 } 289 290 if (cfg) { 291 if (rq->sso_ena) { 292 /* SSO mode */ 293 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 294 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 295 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 296 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 297 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 298 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 299 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 300 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 301 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 302 if (rq->vwqe_ena) { 303 aq->rq_mask.vwqe_ena = ~aq->rq_mask.vwqe_ena; 304 aq->rq_mask.vwqe_skip = ~aq->rq_mask.vwqe_skip; 305 aq->rq_mask.max_vsize_exp = 306 ~aq->rq_mask.max_vsize_exp; 307 aq->rq_mask.vtime_wait = 308 ~aq->rq_mask.vtime_wait; 309 aq->rq_mask.wqe_aura = ~aq->rq_mask.wqe_aura; 310 } 311 } else { 312 /* CQ mode */ 313 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 314 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 315 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 316 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 317 aq->rq_mask.cq = ~aq->rq_mask.cq; 318 } 319 320 if (rq->ipsech_ena) 321 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 322 323 if (rq->spb_ena) { 324 aq->rq_mask.spb_aura = ~aq->rq_mask.spb_aura; 325 aq->rq_mask.spb_sizem1 = ~aq->rq_mask.spb_sizem1; 326 aq->rq_mask.spb_high_sizem1 = 327 ~aq->rq_mask.spb_high_sizem1; 328 } 329 330 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 331 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 332 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 333 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 334 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 335 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 336 aq->rq_mask.ena = ~aq->rq_mask.ena; 337 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 338 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 339 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 340 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 341 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 342 343 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 344 aq->rq_mask.spb_pool_pass = ~aq->rq_mask.spb_pool_pass; 345 aq->rq_mask.lpb_pool_pass = ~aq->rq_mask.lpb_pool_pass; 346 aq->rq_mask.wqe_pool_pass = ~aq->rq_mask.wqe_pool_pass; 347 aq->rq_mask.xqe_pass = ~aq->rq_mask.xqe_pass; 348 349 aq->rq_mask.spb_pool_drop = ~aq->rq_mask.spb_pool_drop; 350 aq->rq_mask.lpb_pool_drop = ~aq->rq_mask.lpb_pool_drop; 351 aq->rq_mask.wqe_pool_drop = ~aq->rq_mask.wqe_pool_drop; 352 aq->rq_mask.xqe_drop = ~aq->rq_mask.xqe_drop; 353 } 354 } 355 356 return 0; 357 } 358 359 int 360 roc_nix_rq_init(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 361 { 362 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 363 struct mbox *mbox = (&nix->dev)->mbox; 364 bool is_cn9k = roc_model_is_cn9k(); 365 struct dev *dev = &nix->dev; 366 int rc; 367 368 if (roc_nix == NULL || rq == NULL) 369 return NIX_ERR_PARAM; 370 371 if (rq->qid >= nix->nb_rx_queues) 372 return NIX_ERR_QUEUE_INVALID_RANGE; 373 374 rq->roc_nix = roc_nix; 375 376 if (is_cn9k) 377 rc = nix_rq_cn9k_cfg(dev, rq, nix->qints, false, ena); 378 else 379 rc = nix_rq_cfg(dev, rq, nix->qints, false, ena); 380 381 if (rc) 382 return rc; 383 384 return mbox_process(mbox); 385 } 386 387 int 388 roc_nix_rq_modify(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 389 { 390 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 391 struct mbox *mbox = (&nix->dev)->mbox; 392 bool is_cn9k = roc_model_is_cn9k(); 393 struct dev *dev = &nix->dev; 394 int rc; 395 396 if (roc_nix == NULL || rq == NULL) 397 return NIX_ERR_PARAM; 398 399 if (rq->qid >= nix->nb_rx_queues) 400 return NIX_ERR_QUEUE_INVALID_RANGE; 401 402 rq->roc_nix = roc_nix; 403 404 if (is_cn9k) 405 rc = nix_rq_cn9k_cfg(dev, rq, nix->qints, true, ena); 406 else 407 rc = nix_rq_cfg(dev, rq, nix->qints, true, ena); 408 409 if (rc) 410 return rc; 411 412 return mbox_process(mbox); 413 } 414 415 int 416 roc_nix_rq_fini(struct roc_nix_rq *rq) 417 { 418 /* Disabling RQ is sufficient */ 419 return roc_nix_rq_ena_dis(rq, false); 420 } 421 422 int 423 roc_nix_cq_init(struct roc_nix *roc_nix, struct roc_nix_cq *cq) 424 { 425 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 426 struct mbox *mbox = (&nix->dev)->mbox; 427 volatile struct nix_cq_ctx_s *cq_ctx; 428 enum nix_q_size qsize; 429 size_t desc_sz; 430 int rc; 431 432 if (cq == NULL) 433 return NIX_ERR_PARAM; 434 435 if (cq->qid >= nix->nb_rx_queues) 436 return NIX_ERR_QUEUE_INVALID_RANGE; 437 438 qsize = nix_qsize_clampup(cq->nb_desc); 439 cq->nb_desc = nix_qsize_to_val(qsize); 440 cq->qmask = cq->nb_desc - 1; 441 cq->door = nix->base + NIX_LF_CQ_OP_DOOR; 442 cq->status = (int64_t *)(nix->base + NIX_LF_CQ_OP_STATUS); 443 cq->wdata = (uint64_t)cq->qid << 32; 444 cq->roc_nix = roc_nix; 445 446 /* CQE of W16 */ 447 desc_sz = cq->nb_desc * NIX_CQ_ENTRY_SZ; 448 cq->desc_base = plt_zmalloc(desc_sz, NIX_CQ_ALIGN); 449 if (cq->desc_base == NULL) { 450 rc = NIX_ERR_NO_MEM; 451 goto fail; 452 } 453 454 if (roc_model_is_cn9k()) { 455 struct nix_aq_enq_req *aq; 456 457 aq = mbox_alloc_msg_nix_aq_enq(mbox); 458 aq->qidx = cq->qid; 459 aq->ctype = NIX_AQ_CTYPE_CQ; 460 aq->op = NIX_AQ_INSTOP_INIT; 461 cq_ctx = &aq->cq; 462 } else { 463 struct nix_cn10k_aq_enq_req *aq; 464 465 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 466 aq->qidx = cq->qid; 467 aq->ctype = NIX_AQ_CTYPE_CQ; 468 aq->op = NIX_AQ_INSTOP_INIT; 469 cq_ctx = &aq->cq; 470 } 471 472 cq_ctx->ena = 1; 473 cq_ctx->caching = 1; 474 cq_ctx->qsize = qsize; 475 cq_ctx->base = (uint64_t)cq->desc_base; 476 cq_ctx->avg_level = 0xff; 477 cq_ctx->cq_err_int_ena = BIT(NIX_CQERRINT_CQE_FAULT); 478 cq_ctx->cq_err_int_ena |= BIT(NIX_CQERRINT_DOOR_ERR); 479 480 /* Many to one reduction */ 481 cq_ctx->qint_idx = cq->qid % nix->qints; 482 /* Map CQ0 [RQ0] to CINT0 and so on till max 64 irqs */ 483 cq_ctx->cint_idx = cq->qid; 484 485 if (roc_model_is_cn96_a0() || roc_model_is_cn95_a0()) { 486 const float rx_cq_skid = NIX_CQ_FULL_ERRATA_SKID; 487 uint16_t min_rx_drop; 488 489 min_rx_drop = ceil(rx_cq_skid / (float)cq->nb_desc); 490 cq_ctx->drop = min_rx_drop; 491 cq_ctx->drop_ena = 1; 492 cq->drop_thresh = min_rx_drop; 493 } else { 494 cq->drop_thresh = NIX_CQ_THRESH_LEVEL; 495 /* Drop processing or red drop cannot be enabled due to 496 * due to packets coming for second pass from CPT. 497 */ 498 if (!roc_nix_inl_inb_is_enabled(roc_nix)) { 499 cq_ctx->drop = cq->drop_thresh; 500 cq_ctx->drop_ena = 1; 501 } 502 } 503 504 /* TX pause frames enable flow ctrl on RX side */ 505 if (nix->tx_pause) { 506 /* Single BPID is allocated for all rx channels for now */ 507 cq_ctx->bpid = nix->bpid[0]; 508 cq_ctx->bp = cq->drop_thresh; 509 cq_ctx->bp_ena = 1; 510 } 511 512 rc = mbox_process(mbox); 513 if (rc) 514 goto free_mem; 515 516 return 0; 517 518 free_mem: 519 plt_free(cq->desc_base); 520 fail: 521 return rc; 522 } 523 524 int 525 roc_nix_cq_fini(struct roc_nix_cq *cq) 526 { 527 struct mbox *mbox; 528 struct nix *nix; 529 int rc; 530 531 if (cq == NULL) 532 return NIX_ERR_PARAM; 533 534 nix = roc_nix_to_nix_priv(cq->roc_nix); 535 mbox = (&nix->dev)->mbox; 536 537 /* Disable CQ */ 538 if (roc_model_is_cn9k()) { 539 struct nix_aq_enq_req *aq; 540 541 aq = mbox_alloc_msg_nix_aq_enq(mbox); 542 aq->qidx = cq->qid; 543 aq->ctype = NIX_AQ_CTYPE_CQ; 544 aq->op = NIX_AQ_INSTOP_WRITE; 545 aq->cq.ena = 0; 546 aq->cq.bp_ena = 0; 547 aq->cq_mask.ena = ~aq->cq_mask.ena; 548 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 549 } else { 550 struct nix_cn10k_aq_enq_req *aq; 551 552 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 553 aq->qidx = cq->qid; 554 aq->ctype = NIX_AQ_CTYPE_CQ; 555 aq->op = NIX_AQ_INSTOP_WRITE; 556 aq->cq.ena = 0; 557 aq->cq.bp_ena = 0; 558 aq->cq_mask.ena = ~aq->cq_mask.ena; 559 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 560 } 561 562 rc = mbox_process(mbox); 563 if (rc) 564 return rc; 565 566 plt_free(cq->desc_base); 567 return 0; 568 } 569 570 static int 571 sqb_pool_populate(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 572 { 573 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 574 uint16_t sqes_per_sqb, count, nb_sqb_bufs; 575 struct npa_pool_s pool; 576 struct npa_aura_s aura; 577 uint64_t blk_sz; 578 uint64_t iova; 579 int rc; 580 581 blk_sz = nix->sqb_size; 582 if (sq->max_sqe_sz == roc_nix_maxsqesz_w16) 583 sqes_per_sqb = (blk_sz / 8) / 16; 584 else 585 sqes_per_sqb = (blk_sz / 8) / 8; 586 587 sq->nb_desc = PLT_MAX(256U, sq->nb_desc); 588 nb_sqb_bufs = sq->nb_desc / sqes_per_sqb; 589 nb_sqb_bufs += NIX_SQB_LIST_SPACE; 590 /* Clamp up the SQB count */ 591 nb_sqb_bufs = PLT_MIN(roc_nix->max_sqb_count, 592 (uint16_t)PLT_MAX(NIX_DEF_SQB, nb_sqb_bufs)); 593 594 sq->nb_sqb_bufs = nb_sqb_bufs; 595 sq->sqes_per_sqb_log2 = (uint16_t)plt_log2_u32(sqes_per_sqb); 596 sq->nb_sqb_bufs_adj = 597 nb_sqb_bufs - 598 (PLT_ALIGN_MUL_CEIL(nb_sqb_bufs, sqes_per_sqb) / sqes_per_sqb); 599 sq->nb_sqb_bufs_adj = 600 (sq->nb_sqb_bufs_adj * NIX_SQB_LOWER_THRESH) / 100; 601 602 /* Explicitly set nat_align alone as by default pool is with both 603 * nat_align and buf_offset = 1 which we don't want for SQB. 604 */ 605 memset(&pool, 0, sizeof(struct npa_pool_s)); 606 pool.nat_align = 1; 607 608 memset(&aura, 0, sizeof(aura)); 609 aura.fc_ena = 1; 610 if (roc_model_is_cn9k() || roc_model_is_cn10ka_a0()) 611 aura.fc_stype = 0x0; /* STF */ 612 else 613 aura.fc_stype = 0x3; /* STSTP */ 614 aura.fc_addr = (uint64_t)sq->fc; 615 aura.fc_hyst_bits = 0; /* Store count on all updates */ 616 rc = roc_npa_pool_create(&sq->aura_handle, blk_sz, NIX_MAX_SQB, &aura, 617 &pool); 618 if (rc) 619 goto fail; 620 621 sq->sqe_mem = plt_zmalloc(blk_sz * NIX_MAX_SQB, blk_sz); 622 if (sq->sqe_mem == NULL) { 623 rc = NIX_ERR_NO_MEM; 624 goto nomem; 625 } 626 627 /* Fill the initial buffers */ 628 iova = (uint64_t)sq->sqe_mem; 629 for (count = 0; count < NIX_MAX_SQB; count++) { 630 roc_npa_aura_op_free(sq->aura_handle, 0, iova); 631 iova += blk_sz; 632 } 633 roc_npa_aura_op_range_set(sq->aura_handle, (uint64_t)sq->sqe_mem, iova); 634 roc_npa_aura_limit_modify(sq->aura_handle, sq->nb_sqb_bufs); 635 sq->aura_sqb_bufs = NIX_MAX_SQB; 636 637 return rc; 638 nomem: 639 roc_npa_pool_destroy(sq->aura_handle); 640 fail: 641 return rc; 642 } 643 644 static void 645 sq_cn9k_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 646 uint16_t smq) 647 { 648 struct mbox *mbox = (&nix->dev)->mbox; 649 struct nix_aq_enq_req *aq; 650 651 aq = mbox_alloc_msg_nix_aq_enq(mbox); 652 aq->qidx = sq->qid; 653 aq->ctype = NIX_AQ_CTYPE_SQ; 654 aq->op = NIX_AQ_INSTOP_INIT; 655 aq->sq.max_sqe_size = sq->max_sqe_sz; 656 657 aq->sq.max_sqe_size = sq->max_sqe_sz; 658 aq->sq.smq = smq; 659 aq->sq.smq_rr_quantum = rr_quantum; 660 aq->sq.default_chan = nix->tx_chan_base; 661 aq->sq.sqe_stype = NIX_STYPE_STF; 662 aq->sq.ena = 1; 663 aq->sq.sso_ena = !!sq->sso_ena; 664 aq->sq.cq_ena = !!sq->cq_ena; 665 aq->sq.cq = sq->cqid; 666 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 667 aq->sq.sqe_stype = NIX_STYPE_STP; 668 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 669 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 670 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 671 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 672 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 673 674 /* Many to one reduction */ 675 aq->sq.qint_idx = sq->qid % nix->qints; 676 } 677 678 static int 679 sq_cn9k_fini(struct nix *nix, struct roc_nix_sq *sq) 680 { 681 struct mbox *mbox = (&nix->dev)->mbox; 682 struct nix_aq_enq_rsp *rsp; 683 struct nix_aq_enq_req *aq; 684 uint16_t sqes_per_sqb; 685 void *sqb_buf; 686 int rc, count; 687 688 aq = mbox_alloc_msg_nix_aq_enq(mbox); 689 aq->qidx = sq->qid; 690 aq->ctype = NIX_AQ_CTYPE_SQ; 691 aq->op = NIX_AQ_INSTOP_READ; 692 rc = mbox_process_msg(mbox, (void *)&rsp); 693 if (rc) 694 return rc; 695 696 /* Check if sq is already cleaned up */ 697 if (!rsp->sq.ena) 698 return 0; 699 700 /* Disable sq */ 701 aq = mbox_alloc_msg_nix_aq_enq(mbox); 702 aq->qidx = sq->qid; 703 aq->ctype = NIX_AQ_CTYPE_SQ; 704 aq->op = NIX_AQ_INSTOP_WRITE; 705 aq->sq_mask.ena = ~aq->sq_mask.ena; 706 aq->sq.ena = 0; 707 rc = mbox_process(mbox); 708 if (rc) 709 return rc; 710 711 /* Read SQ and free sqb's */ 712 aq = mbox_alloc_msg_nix_aq_enq(mbox); 713 aq->qidx = sq->qid; 714 aq->ctype = NIX_AQ_CTYPE_SQ; 715 aq->op = NIX_AQ_INSTOP_READ; 716 rc = mbox_process_msg(mbox, (void *)&rsp); 717 if (rc) 718 return rc; 719 720 if (aq->sq.smq_pend) 721 plt_err("SQ has pending SQE's"); 722 723 count = aq->sq.sqb_count; 724 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 725 /* Free SQB's that are used */ 726 sqb_buf = (void *)rsp->sq.head_sqb; 727 while (count) { 728 void *next_sqb; 729 730 next_sqb = *(void **)((uintptr_t)sqb_buf + 731 (uint32_t)((sqes_per_sqb - 1) * 732 sq->max_sqe_sz)); 733 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 734 sqb_buf = next_sqb; 735 count--; 736 } 737 738 /* Free next to use sqb */ 739 if (rsp->sq.next_sqb) 740 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 741 return 0; 742 } 743 744 static void 745 sq_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 746 uint16_t smq) 747 { 748 struct mbox *mbox = (&nix->dev)->mbox; 749 struct nix_cn10k_aq_enq_req *aq; 750 751 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 752 aq->qidx = sq->qid; 753 aq->ctype = NIX_AQ_CTYPE_SQ; 754 aq->op = NIX_AQ_INSTOP_INIT; 755 aq->sq.max_sqe_size = sq->max_sqe_sz; 756 757 aq->sq.max_sqe_size = sq->max_sqe_sz; 758 aq->sq.smq = smq; 759 aq->sq.smq_rr_weight = rr_quantum; 760 aq->sq.default_chan = nix->tx_chan_base; 761 aq->sq.sqe_stype = NIX_STYPE_STF; 762 aq->sq.ena = 1; 763 aq->sq.sso_ena = !!sq->sso_ena; 764 aq->sq.cq_ena = !!sq->cq_ena; 765 aq->sq.cq = sq->cqid; 766 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 767 aq->sq.sqe_stype = NIX_STYPE_STP; 768 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 769 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 770 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 771 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 772 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 773 774 /* Many to one reduction */ 775 aq->sq.qint_idx = sq->qid % nix->qints; 776 } 777 778 static int 779 sq_fini(struct nix *nix, struct roc_nix_sq *sq) 780 { 781 struct mbox *mbox = (&nix->dev)->mbox; 782 struct nix_cn10k_aq_enq_rsp *rsp; 783 struct nix_cn10k_aq_enq_req *aq; 784 uint16_t sqes_per_sqb; 785 void *sqb_buf; 786 int rc, count; 787 788 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 789 aq->qidx = sq->qid; 790 aq->ctype = NIX_AQ_CTYPE_SQ; 791 aq->op = NIX_AQ_INSTOP_READ; 792 rc = mbox_process_msg(mbox, (void *)&rsp); 793 if (rc) 794 return rc; 795 796 /* Check if sq is already cleaned up */ 797 if (!rsp->sq.ena) 798 return 0; 799 800 /* Disable sq */ 801 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 802 aq->qidx = sq->qid; 803 aq->ctype = NIX_AQ_CTYPE_SQ; 804 aq->op = NIX_AQ_INSTOP_WRITE; 805 aq->sq_mask.ena = ~aq->sq_mask.ena; 806 aq->sq.ena = 0; 807 rc = mbox_process(mbox); 808 if (rc) 809 return rc; 810 811 /* Read SQ and free sqb's */ 812 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 813 aq->qidx = sq->qid; 814 aq->ctype = NIX_AQ_CTYPE_SQ; 815 aq->op = NIX_AQ_INSTOP_READ; 816 rc = mbox_process_msg(mbox, (void *)&rsp); 817 if (rc) 818 return rc; 819 820 if (aq->sq.smq_pend) 821 plt_err("SQ has pending SQE's"); 822 823 count = aq->sq.sqb_count; 824 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 825 /* Free SQB's that are used */ 826 sqb_buf = (void *)rsp->sq.head_sqb; 827 while (count) { 828 void *next_sqb; 829 830 next_sqb = *(void **)((uintptr_t)sqb_buf + 831 (uint32_t)((sqes_per_sqb - 1) * 832 sq->max_sqe_sz)); 833 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 834 sqb_buf = next_sqb; 835 count--; 836 } 837 838 /* Free next to use sqb */ 839 if (rsp->sq.next_sqb) 840 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 841 return 0; 842 } 843 844 int 845 roc_nix_sq_init(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 846 { 847 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 848 struct mbox *mbox = (&nix->dev)->mbox; 849 uint16_t qid, smq = UINT16_MAX; 850 uint32_t rr_quantum = 0; 851 int rc; 852 853 if (sq == NULL) 854 return NIX_ERR_PARAM; 855 856 qid = sq->qid; 857 if (qid >= nix->nb_tx_queues) 858 return NIX_ERR_QUEUE_INVALID_RANGE; 859 860 sq->roc_nix = roc_nix; 861 /* 862 * Allocate memory for flow control updates from HW. 863 * Alloc one cache line, so that fits all FC_STYPE modes. 864 */ 865 sq->fc = plt_zmalloc(ROC_ALIGN, ROC_ALIGN); 866 if (sq->fc == NULL) { 867 rc = NIX_ERR_NO_MEM; 868 goto fail; 869 } 870 871 rc = sqb_pool_populate(roc_nix, sq); 872 if (rc) 873 goto nomem; 874 875 rc = nix_tm_leaf_data_get(nix, sq->qid, &rr_quantum, &smq); 876 if (rc) { 877 rc = NIX_ERR_TM_LEAF_NODE_GET; 878 goto nomem; 879 } 880 881 /* Init SQ context */ 882 if (roc_model_is_cn9k()) 883 sq_cn9k_init(nix, sq, rr_quantum, smq); 884 else 885 sq_init(nix, sq, rr_quantum, smq); 886 887 rc = mbox_process(mbox); 888 if (rc) 889 goto nomem; 890 891 nix->sqs[qid] = sq; 892 sq->io_addr = nix->base + NIX_LF_OP_SENDX(0); 893 /* Evenly distribute LMT slot for each sq */ 894 if (roc_model_is_cn9k()) { 895 /* Multiple cores/SQ's can use same LMTLINE safely in CN9K */ 896 sq->lmt_addr = (void *)(nix->lmt_base + 897 ((qid & RVU_CN9K_LMT_SLOT_MASK) << 12)); 898 } 899 900 return rc; 901 nomem: 902 plt_free(sq->fc); 903 fail: 904 return rc; 905 } 906 907 int 908 roc_nix_sq_fini(struct roc_nix_sq *sq) 909 { 910 struct nix *nix; 911 struct mbox *mbox; 912 struct ndc_sync_op *ndc_req; 913 uint16_t qid; 914 int rc = 0; 915 916 if (sq == NULL) 917 return NIX_ERR_PARAM; 918 919 nix = roc_nix_to_nix_priv(sq->roc_nix); 920 mbox = (&nix->dev)->mbox; 921 922 qid = sq->qid; 923 924 rc = nix_tm_sq_flush_pre(sq); 925 926 /* Release SQ context */ 927 if (roc_model_is_cn9k()) 928 rc |= sq_cn9k_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 929 else 930 rc |= sq_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 931 932 /* Sync NDC-NIX-TX for LF */ 933 ndc_req = mbox_alloc_msg_ndc_sync_op(mbox); 934 if (ndc_req == NULL) 935 return -ENOSPC; 936 ndc_req->nix_lf_tx_sync = 1; 937 if (mbox_process(mbox)) 938 rc |= NIX_ERR_NDC_SYNC; 939 940 rc |= nix_tm_sq_flush_post(sq); 941 942 /* Restore limit to max SQB count that the pool was created 943 * for aura drain to succeed. 944 */ 945 roc_npa_aura_limit_modify(sq->aura_handle, NIX_MAX_SQB); 946 rc |= roc_npa_pool_destroy(sq->aura_handle); 947 plt_free(sq->fc); 948 plt_free(sq->sqe_mem); 949 nix->sqs[qid] = NULL; 950 951 return rc; 952 } 953