1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include "roc_api.h" 6 #include "roc_priv.h" 7 8 static inline uint32_t 9 nix_qsize_to_val(enum nix_q_size qsize) 10 { 11 return (16UL << (qsize * 2)); 12 } 13 14 static inline enum nix_q_size 15 nix_qsize_clampup(uint32_t val) 16 { 17 int i = nix_q_size_16; 18 19 for (; i < nix_q_size_max; i++) 20 if (val <= nix_qsize_to_val(i)) 21 break; 22 23 if (i >= nix_q_size_max) 24 i = nix_q_size_max - 1; 25 26 return i; 27 } 28 29 int 30 roc_nix_rq_ena_dis(struct roc_nix_rq *rq, bool enable) 31 { 32 struct nix *nix = roc_nix_to_nix_priv(rq->roc_nix); 33 struct mbox *mbox = (&nix->dev)->mbox; 34 int rc; 35 36 /* Pkts will be dropped silently if RQ is disabled */ 37 if (roc_model_is_cn9k()) { 38 struct nix_aq_enq_req *aq; 39 40 aq = mbox_alloc_msg_nix_aq_enq(mbox); 41 aq->qidx = rq->qid; 42 aq->ctype = NIX_AQ_CTYPE_RQ; 43 aq->op = NIX_AQ_INSTOP_WRITE; 44 45 aq->rq.ena = enable; 46 aq->rq_mask.ena = ~(aq->rq_mask.ena); 47 } else { 48 struct nix_cn10k_aq_enq_req *aq; 49 50 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 51 aq->qidx = rq->qid; 52 aq->ctype = NIX_AQ_CTYPE_RQ; 53 aq->op = NIX_AQ_INSTOP_WRITE; 54 55 aq->rq.ena = enable; 56 aq->rq_mask.ena = ~(aq->rq_mask.ena); 57 } 58 59 rc = mbox_process(mbox); 60 61 if (roc_model_is_cn10k()) 62 plt_write64(rq->qid, nix->base + NIX_LF_OP_VWQE_FLUSH); 63 return rc; 64 } 65 66 static int 67 rq_cn9k_cfg(struct nix *nix, struct roc_nix_rq *rq, bool cfg, bool ena) 68 { 69 struct mbox *mbox = (&nix->dev)->mbox; 70 struct nix_aq_enq_req *aq; 71 72 aq = mbox_alloc_msg_nix_aq_enq(mbox); 73 aq->qidx = rq->qid; 74 aq->ctype = NIX_AQ_CTYPE_RQ; 75 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 76 77 if (rq->sso_ena) { 78 /* SSO mode */ 79 aq->rq.sso_ena = 1; 80 aq->rq.sso_tt = rq->tt; 81 aq->rq.sso_grp = rq->hwgrp; 82 aq->rq.ena_wqwd = 1; 83 aq->rq.wqe_skip = rq->wqe_skip; 84 aq->rq.wqe_caching = 1; 85 86 aq->rq.good_utag = rq->tag_mask >> 24; 87 aq->rq.bad_utag = rq->tag_mask >> 24; 88 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 89 } else { 90 /* CQ mode */ 91 aq->rq.sso_ena = 0; 92 aq->rq.good_utag = rq->tag_mask >> 24; 93 aq->rq.bad_utag = rq->tag_mask >> 24; 94 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 95 aq->rq.cq = rq->qid; 96 } 97 98 if (rq->ipsech_ena) 99 aq->rq.ipsech_ena = 1; 100 101 aq->rq.spb_ena = 0; 102 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 103 104 /* Sizes must be aligned to 8 bytes */ 105 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 106 return -EINVAL; 107 108 /* Expressed in number of dwords */ 109 aq->rq.first_skip = rq->first_skip / 8; 110 aq->rq.later_skip = rq->later_skip / 8; 111 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 112 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 113 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 114 aq->rq.ena = ena; 115 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 116 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 117 aq->rq.rq_int_ena = 0; 118 /* Many to one reduction */ 119 aq->rq.qint_idx = rq->qid % nix->qints; 120 aq->rq.xqe_drop_ena = 1; 121 122 if (cfg) { 123 if (rq->sso_ena) { 124 /* SSO mode */ 125 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 126 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 127 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 128 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 129 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 130 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 131 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 132 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 133 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 134 } else { 135 /* CQ mode */ 136 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 137 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 138 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 139 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 140 aq->rq_mask.cq = ~aq->rq_mask.cq; 141 } 142 143 if (rq->ipsech_ena) 144 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 145 146 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 147 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 148 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 149 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 150 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 151 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 152 aq->rq_mask.ena = ~aq->rq_mask.ena; 153 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 154 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 155 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 156 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 157 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 158 } 159 160 return 0; 161 } 162 163 static int 164 rq_cfg(struct nix *nix, struct roc_nix_rq *rq, bool cfg, bool ena) 165 { 166 struct mbox *mbox = (&nix->dev)->mbox; 167 struct nix_cn10k_aq_enq_req *aq; 168 169 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 170 aq->qidx = rq->qid; 171 aq->ctype = NIX_AQ_CTYPE_RQ; 172 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 173 174 if (rq->sso_ena) { 175 /* SSO mode */ 176 aq->rq.sso_ena = 1; 177 aq->rq.sso_tt = rq->tt; 178 aq->rq.sso_grp = rq->hwgrp; 179 aq->rq.ena_wqwd = 1; 180 aq->rq.wqe_skip = rq->wqe_skip; 181 aq->rq.wqe_caching = 1; 182 183 aq->rq.good_utag = rq->tag_mask >> 24; 184 aq->rq.bad_utag = rq->tag_mask >> 24; 185 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 186 187 if (rq->vwqe_ena) { 188 aq->rq.vwqe_ena = true; 189 aq->rq.vwqe_skip = rq->vwqe_first_skip; 190 /* Maximal Vector size is (2^(MAX_VSIZE_EXP+2)) */ 191 aq->rq.max_vsize_exp = rq->vwqe_max_sz_exp - 2; 192 aq->rq.vtime_wait = rq->vwqe_wait_tmo; 193 aq->rq.wqe_aura = rq->vwqe_aura_handle; 194 } 195 } else { 196 /* CQ mode */ 197 aq->rq.sso_ena = 0; 198 aq->rq.good_utag = rq->tag_mask >> 24; 199 aq->rq.bad_utag = rq->tag_mask >> 24; 200 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 201 aq->rq.cq = rq->qid; 202 } 203 204 if (rq->ipsech_ena) 205 aq->rq.ipsech_ena = 1; 206 207 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 208 209 /* Sizes must be aligned to 8 bytes */ 210 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 211 return -EINVAL; 212 213 /* Expressed in number of dwords */ 214 aq->rq.first_skip = rq->first_skip / 8; 215 aq->rq.later_skip = rq->later_skip / 8; 216 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 217 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 218 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 219 aq->rq.ena = ena; 220 221 if (rq->spb_ena) { 222 uint32_t spb_sizem1; 223 224 aq->rq.spb_ena = 1; 225 aq->rq.spb_aura = 226 roc_npa_aura_handle_to_aura(rq->spb_aura_handle); 227 228 if (rq->spb_size & 0x7 || 229 rq->spb_size > NIX_RQ_CN10K_SPB_MAX_SIZE) 230 return -EINVAL; 231 232 spb_sizem1 = rq->spb_size / 8; /* Expressed in no. of dwords */ 233 spb_sizem1 -= 1; /* Expressed in size minus one */ 234 aq->rq.spb_sizem1 = spb_sizem1 & 0x3F; 235 aq->rq.spb_high_sizem1 = (spb_sizem1 >> 6) & 0x7; 236 } else { 237 aq->rq.spb_ena = 0; 238 } 239 240 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 241 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 242 aq->rq.rq_int_ena = 0; 243 /* Many to one reduction */ 244 aq->rq.qint_idx = rq->qid % nix->qints; 245 aq->rq.xqe_drop_ena = 1; 246 247 if (cfg) { 248 if (rq->sso_ena) { 249 /* SSO mode */ 250 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 251 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 252 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 253 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 254 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 255 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 256 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 257 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 258 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 259 if (rq->vwqe_ena) { 260 aq->rq_mask.vwqe_ena = ~aq->rq_mask.vwqe_ena; 261 aq->rq_mask.vwqe_skip = ~aq->rq_mask.vwqe_skip; 262 aq->rq_mask.max_vsize_exp = 263 ~aq->rq_mask.max_vsize_exp; 264 aq->rq_mask.vtime_wait = 265 ~aq->rq_mask.vtime_wait; 266 aq->rq_mask.wqe_aura = ~aq->rq_mask.wqe_aura; 267 } 268 } else { 269 /* CQ mode */ 270 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 271 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 272 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 273 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 274 aq->rq_mask.cq = ~aq->rq_mask.cq; 275 } 276 277 if (rq->ipsech_ena) 278 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 279 280 if (rq->spb_ena) { 281 aq->rq_mask.spb_aura = ~aq->rq_mask.spb_aura; 282 aq->rq_mask.spb_sizem1 = ~aq->rq_mask.spb_sizem1; 283 aq->rq_mask.spb_high_sizem1 = 284 ~aq->rq_mask.spb_high_sizem1; 285 } 286 287 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 288 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 289 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 290 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 291 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 292 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 293 aq->rq_mask.ena = ~aq->rq_mask.ena; 294 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 295 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 296 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 297 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 298 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 299 } 300 301 return 0; 302 } 303 304 int 305 roc_nix_rq_init(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 306 { 307 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 308 struct mbox *mbox = (&nix->dev)->mbox; 309 bool is_cn9k = roc_model_is_cn9k(); 310 int rc; 311 312 if (roc_nix == NULL || rq == NULL) 313 return NIX_ERR_PARAM; 314 315 if (rq->qid >= nix->nb_rx_queues) 316 return NIX_ERR_QUEUE_INVALID_RANGE; 317 318 rq->roc_nix = roc_nix; 319 320 if (is_cn9k) 321 rc = rq_cn9k_cfg(nix, rq, false, ena); 322 else 323 rc = rq_cfg(nix, rq, false, ena); 324 325 if (rc) 326 return rc; 327 328 return mbox_process(mbox); 329 } 330 331 int 332 roc_nix_rq_modify(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 333 { 334 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 335 struct mbox *mbox = (&nix->dev)->mbox; 336 bool is_cn9k = roc_model_is_cn9k(); 337 int rc; 338 339 if (roc_nix == NULL || rq == NULL) 340 return NIX_ERR_PARAM; 341 342 if (rq->qid >= nix->nb_rx_queues) 343 return NIX_ERR_QUEUE_INVALID_RANGE; 344 345 rq->roc_nix = roc_nix; 346 347 if (is_cn9k) 348 rc = rq_cn9k_cfg(nix, rq, true, ena); 349 else 350 rc = rq_cfg(nix, rq, true, ena); 351 352 if (rc) 353 return rc; 354 355 return mbox_process(mbox); 356 } 357 358 int 359 roc_nix_rq_fini(struct roc_nix_rq *rq) 360 { 361 /* Disabling RQ is sufficient */ 362 return roc_nix_rq_ena_dis(rq, false); 363 } 364 365 int 366 roc_nix_cq_init(struct roc_nix *roc_nix, struct roc_nix_cq *cq) 367 { 368 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 369 struct mbox *mbox = (&nix->dev)->mbox; 370 volatile struct nix_cq_ctx_s *cq_ctx; 371 enum nix_q_size qsize; 372 size_t desc_sz; 373 int rc; 374 375 if (cq == NULL) 376 return NIX_ERR_PARAM; 377 378 if (cq->qid >= nix->nb_rx_queues) 379 return NIX_ERR_QUEUE_INVALID_RANGE; 380 381 qsize = nix_qsize_clampup(cq->nb_desc); 382 cq->nb_desc = nix_qsize_to_val(qsize); 383 cq->qmask = cq->nb_desc - 1; 384 cq->door = nix->base + NIX_LF_CQ_OP_DOOR; 385 cq->status = (int64_t *)(nix->base + NIX_LF_CQ_OP_STATUS); 386 cq->wdata = (uint64_t)cq->qid << 32; 387 cq->roc_nix = roc_nix; 388 cq->drop_thresh = NIX_CQ_THRESH_LEVEL; 389 390 /* CQE of W16 */ 391 desc_sz = cq->nb_desc * NIX_CQ_ENTRY_SZ; 392 cq->desc_base = plt_zmalloc(desc_sz, NIX_CQ_ALIGN); 393 if (cq->desc_base == NULL) { 394 rc = NIX_ERR_NO_MEM; 395 goto fail; 396 } 397 398 if (roc_model_is_cn9k()) { 399 struct nix_aq_enq_req *aq; 400 401 aq = mbox_alloc_msg_nix_aq_enq(mbox); 402 aq->qidx = cq->qid; 403 aq->ctype = NIX_AQ_CTYPE_CQ; 404 aq->op = NIX_AQ_INSTOP_INIT; 405 cq_ctx = &aq->cq; 406 } else { 407 struct nix_cn10k_aq_enq_req *aq; 408 409 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 410 aq->qidx = cq->qid; 411 aq->ctype = NIX_AQ_CTYPE_CQ; 412 aq->op = NIX_AQ_INSTOP_INIT; 413 cq_ctx = &aq->cq; 414 } 415 416 cq_ctx->ena = 1; 417 cq_ctx->caching = 1; 418 cq_ctx->qsize = qsize; 419 cq_ctx->base = (uint64_t)cq->desc_base; 420 cq_ctx->avg_level = 0xff; 421 cq_ctx->cq_err_int_ena = BIT(NIX_CQERRINT_CQE_FAULT); 422 cq_ctx->cq_err_int_ena |= BIT(NIX_CQERRINT_DOOR_ERR); 423 424 /* Many to one reduction */ 425 cq_ctx->qint_idx = cq->qid % nix->qints; 426 /* Map CQ0 [RQ0] to CINT0 and so on till max 64 irqs */ 427 cq_ctx->cint_idx = cq->qid; 428 429 cq_ctx->drop = cq->drop_thresh; 430 cq_ctx->drop_ena = 1; 431 432 /* TX pause frames enable flow ctrl on RX side */ 433 if (nix->tx_pause) { 434 /* Single BPID is allocated for all rx channels for now */ 435 cq_ctx->bpid = nix->bpid[0]; 436 cq_ctx->bp = cq_ctx->drop; 437 cq_ctx->bp_ena = 1; 438 } 439 440 rc = mbox_process(mbox); 441 if (rc) 442 goto free_mem; 443 444 return 0; 445 446 free_mem: 447 plt_free(cq->desc_base); 448 fail: 449 return rc; 450 } 451 452 int 453 roc_nix_cq_fini(struct roc_nix_cq *cq) 454 { 455 struct mbox *mbox; 456 struct nix *nix; 457 int rc; 458 459 if (cq == NULL) 460 return NIX_ERR_PARAM; 461 462 nix = roc_nix_to_nix_priv(cq->roc_nix); 463 mbox = (&nix->dev)->mbox; 464 465 /* Disable CQ */ 466 if (roc_model_is_cn9k()) { 467 struct nix_aq_enq_req *aq; 468 469 aq = mbox_alloc_msg_nix_aq_enq(mbox); 470 aq->qidx = cq->qid; 471 aq->ctype = NIX_AQ_CTYPE_CQ; 472 aq->op = NIX_AQ_INSTOP_WRITE; 473 aq->cq.ena = 0; 474 aq->cq.bp_ena = 0; 475 aq->cq_mask.ena = ~aq->cq_mask.ena; 476 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 477 } else { 478 struct nix_cn10k_aq_enq_req *aq; 479 480 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 481 aq->qidx = cq->qid; 482 aq->ctype = NIX_AQ_CTYPE_CQ; 483 aq->op = NIX_AQ_INSTOP_WRITE; 484 aq->cq.ena = 0; 485 aq->cq.bp_ena = 0; 486 aq->cq_mask.ena = ~aq->cq_mask.ena; 487 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 488 } 489 490 rc = mbox_process(mbox); 491 if (rc) 492 return rc; 493 494 plt_free(cq->desc_base); 495 return 0; 496 } 497 498 static int 499 sqb_pool_populate(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 500 { 501 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 502 uint16_t sqes_per_sqb, count, nb_sqb_bufs; 503 struct npa_pool_s pool; 504 struct npa_aura_s aura; 505 uint64_t blk_sz; 506 uint64_t iova; 507 int rc; 508 509 blk_sz = nix->sqb_size; 510 if (sq->max_sqe_sz == roc_nix_maxsqesz_w16) 511 sqes_per_sqb = (blk_sz / 8) / 16; 512 else 513 sqes_per_sqb = (blk_sz / 8) / 8; 514 515 sq->nb_desc = PLT_MAX(256U, sq->nb_desc); 516 nb_sqb_bufs = sq->nb_desc / sqes_per_sqb; 517 nb_sqb_bufs += NIX_SQB_LIST_SPACE; 518 /* Clamp up the SQB count */ 519 nb_sqb_bufs = PLT_MIN(roc_nix->max_sqb_count, 520 (uint16_t)PLT_MAX(NIX_DEF_SQB, nb_sqb_bufs)); 521 522 sq->nb_sqb_bufs = nb_sqb_bufs; 523 sq->sqes_per_sqb_log2 = (uint16_t)plt_log2_u32(sqes_per_sqb); 524 sq->nb_sqb_bufs_adj = 525 nb_sqb_bufs - 526 (PLT_ALIGN_MUL_CEIL(nb_sqb_bufs, sqes_per_sqb) / sqes_per_sqb); 527 sq->nb_sqb_bufs_adj = 528 (sq->nb_sqb_bufs_adj * NIX_SQB_LOWER_THRESH) / 100; 529 530 /* Explicitly set nat_align alone as by default pool is with both 531 * nat_align and buf_offset = 1 which we don't want for SQB. 532 */ 533 memset(&pool, 0, sizeof(struct npa_pool_s)); 534 pool.nat_align = 1; 535 536 memset(&aura, 0, sizeof(aura)); 537 aura.fc_ena = 1; 538 aura.fc_addr = (uint64_t)sq->fc; 539 aura.fc_hyst_bits = 0; /* Store count on all updates */ 540 rc = roc_npa_pool_create(&sq->aura_handle, blk_sz, nb_sqb_bufs, &aura, 541 &pool); 542 if (rc) 543 goto fail; 544 545 sq->sqe_mem = plt_zmalloc(blk_sz * nb_sqb_bufs, blk_sz); 546 if (sq->sqe_mem == NULL) { 547 rc = NIX_ERR_NO_MEM; 548 goto nomem; 549 } 550 551 /* Fill the initial buffers */ 552 iova = (uint64_t)sq->sqe_mem; 553 for (count = 0; count < nb_sqb_bufs; count++) { 554 roc_npa_aura_op_free(sq->aura_handle, 0, iova); 555 iova += blk_sz; 556 } 557 roc_npa_aura_op_range_set(sq->aura_handle, (uint64_t)sq->sqe_mem, iova); 558 559 return rc; 560 nomem: 561 roc_npa_pool_destroy(sq->aura_handle); 562 fail: 563 return rc; 564 } 565 566 static void 567 sq_cn9k_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 568 uint16_t smq) 569 { 570 struct mbox *mbox = (&nix->dev)->mbox; 571 struct nix_aq_enq_req *aq; 572 573 aq = mbox_alloc_msg_nix_aq_enq(mbox); 574 aq->qidx = sq->qid; 575 aq->ctype = NIX_AQ_CTYPE_SQ; 576 aq->op = NIX_AQ_INSTOP_INIT; 577 aq->sq.max_sqe_size = sq->max_sqe_sz; 578 579 aq->sq.max_sqe_size = sq->max_sqe_sz; 580 aq->sq.smq = smq; 581 aq->sq.smq_rr_quantum = rr_quantum; 582 aq->sq.default_chan = nix->tx_chan_base; 583 aq->sq.sqe_stype = NIX_STYPE_STF; 584 aq->sq.ena = 1; 585 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 586 aq->sq.sqe_stype = NIX_STYPE_STP; 587 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 588 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 589 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 590 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 591 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 592 593 /* Many to one reduction */ 594 aq->sq.qint_idx = sq->qid % nix->qints; 595 } 596 597 static int 598 sq_cn9k_fini(struct nix *nix, struct roc_nix_sq *sq) 599 { 600 struct mbox *mbox = (&nix->dev)->mbox; 601 struct nix_aq_enq_rsp *rsp; 602 struct nix_aq_enq_req *aq; 603 uint16_t sqes_per_sqb; 604 void *sqb_buf; 605 int rc, count; 606 607 aq = mbox_alloc_msg_nix_aq_enq(mbox); 608 aq->qidx = sq->qid; 609 aq->ctype = NIX_AQ_CTYPE_SQ; 610 aq->op = NIX_AQ_INSTOP_READ; 611 rc = mbox_process_msg(mbox, (void *)&rsp); 612 if (rc) 613 return rc; 614 615 /* Check if sq is already cleaned up */ 616 if (!rsp->sq.ena) 617 return 0; 618 619 /* Disable sq */ 620 aq = mbox_alloc_msg_nix_aq_enq(mbox); 621 aq->qidx = sq->qid; 622 aq->ctype = NIX_AQ_CTYPE_SQ; 623 aq->op = NIX_AQ_INSTOP_WRITE; 624 aq->sq_mask.ena = ~aq->sq_mask.ena; 625 aq->sq.ena = 0; 626 rc = mbox_process(mbox); 627 if (rc) 628 return rc; 629 630 /* Read SQ and free sqb's */ 631 aq = mbox_alloc_msg_nix_aq_enq(mbox); 632 aq->qidx = sq->qid; 633 aq->ctype = NIX_AQ_CTYPE_SQ; 634 aq->op = NIX_AQ_INSTOP_READ; 635 rc = mbox_process_msg(mbox, (void *)&rsp); 636 if (rc) 637 return rc; 638 639 if (aq->sq.smq_pend) 640 plt_err("SQ has pending SQE's"); 641 642 count = aq->sq.sqb_count; 643 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 644 /* Free SQB's that are used */ 645 sqb_buf = (void *)rsp->sq.head_sqb; 646 while (count) { 647 void *next_sqb; 648 649 next_sqb = *(void **)((uintptr_t)sqb_buf + 650 (uint32_t)((sqes_per_sqb - 1) * 651 sq->max_sqe_sz)); 652 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 653 sqb_buf = next_sqb; 654 count--; 655 } 656 657 /* Free next to use sqb */ 658 if (rsp->sq.next_sqb) 659 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 660 return 0; 661 } 662 663 static void 664 sq_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 665 uint16_t smq) 666 { 667 struct mbox *mbox = (&nix->dev)->mbox; 668 struct nix_cn10k_aq_enq_req *aq; 669 670 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 671 aq->qidx = sq->qid; 672 aq->ctype = NIX_AQ_CTYPE_SQ; 673 aq->op = NIX_AQ_INSTOP_INIT; 674 aq->sq.max_sqe_size = sq->max_sqe_sz; 675 676 aq->sq.max_sqe_size = sq->max_sqe_sz; 677 aq->sq.smq = smq; 678 aq->sq.smq_rr_weight = rr_quantum; 679 aq->sq.default_chan = nix->tx_chan_base; 680 aq->sq.sqe_stype = NIX_STYPE_STF; 681 aq->sq.ena = 1; 682 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 683 aq->sq.sqe_stype = NIX_STYPE_STP; 684 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 685 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 686 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 687 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 688 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 689 690 /* Many to one reduction */ 691 aq->sq.qint_idx = sq->qid % nix->qints; 692 } 693 694 static int 695 sq_fini(struct nix *nix, struct roc_nix_sq *sq) 696 { 697 struct mbox *mbox = (&nix->dev)->mbox; 698 struct nix_cn10k_aq_enq_rsp *rsp; 699 struct nix_cn10k_aq_enq_req *aq; 700 uint16_t sqes_per_sqb; 701 void *sqb_buf; 702 int rc, count; 703 704 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 705 aq->qidx = sq->qid; 706 aq->ctype = NIX_AQ_CTYPE_SQ; 707 aq->op = NIX_AQ_INSTOP_READ; 708 rc = mbox_process_msg(mbox, (void *)&rsp); 709 if (rc) 710 return rc; 711 712 /* Check if sq is already cleaned up */ 713 if (!rsp->sq.ena) 714 return 0; 715 716 /* Disable sq */ 717 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 718 aq->qidx = sq->qid; 719 aq->ctype = NIX_AQ_CTYPE_SQ; 720 aq->op = NIX_AQ_INSTOP_WRITE; 721 aq->sq_mask.ena = ~aq->sq_mask.ena; 722 aq->sq.ena = 0; 723 rc = mbox_process(mbox); 724 if (rc) 725 return rc; 726 727 /* Read SQ and free sqb's */ 728 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 729 aq->qidx = sq->qid; 730 aq->ctype = NIX_AQ_CTYPE_SQ; 731 aq->op = NIX_AQ_INSTOP_READ; 732 rc = mbox_process_msg(mbox, (void *)&rsp); 733 if (rc) 734 return rc; 735 736 if (aq->sq.smq_pend) 737 plt_err("SQ has pending SQE's"); 738 739 count = aq->sq.sqb_count; 740 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 741 /* Free SQB's that are used */ 742 sqb_buf = (void *)rsp->sq.head_sqb; 743 while (count) { 744 void *next_sqb; 745 746 next_sqb = *(void **)((uintptr_t)sqb_buf + 747 (uint32_t)((sqes_per_sqb - 1) * 748 sq->max_sqe_sz)); 749 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 750 sqb_buf = next_sqb; 751 count--; 752 } 753 754 /* Free next to use sqb */ 755 if (rsp->sq.next_sqb) 756 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 757 return 0; 758 } 759 760 int 761 roc_nix_sq_init(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 762 { 763 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 764 struct mbox *mbox = (&nix->dev)->mbox; 765 uint16_t qid, smq = UINT16_MAX; 766 uint32_t rr_quantum = 0; 767 int rc; 768 769 if (sq == NULL) 770 return NIX_ERR_PARAM; 771 772 qid = sq->qid; 773 if (qid >= nix->nb_tx_queues) 774 return NIX_ERR_QUEUE_INVALID_RANGE; 775 776 sq->roc_nix = roc_nix; 777 /* 778 * Allocate memory for flow control updates from HW. 779 * Alloc one cache line, so that fits all FC_STYPE modes. 780 */ 781 sq->fc = plt_zmalloc(ROC_ALIGN, ROC_ALIGN); 782 if (sq->fc == NULL) { 783 rc = NIX_ERR_NO_MEM; 784 goto fail; 785 } 786 787 rc = sqb_pool_populate(roc_nix, sq); 788 if (rc) 789 goto nomem; 790 791 rc = nix_tm_leaf_data_get(nix, sq->qid, &rr_quantum, &smq); 792 if (rc) { 793 rc = NIX_ERR_TM_LEAF_NODE_GET; 794 goto nomem; 795 } 796 797 /* Init SQ context */ 798 if (roc_model_is_cn9k()) 799 sq_cn9k_init(nix, sq, rr_quantum, smq); 800 else 801 sq_init(nix, sq, rr_quantum, smq); 802 803 rc = mbox_process(mbox); 804 if (rc) 805 goto nomem; 806 807 nix->sqs[qid] = sq; 808 sq->io_addr = nix->base + NIX_LF_OP_SENDX(0); 809 /* Evenly distribute LMT slot for each sq */ 810 if (roc_model_is_cn9k()) { 811 /* Multiple cores/SQ's can use same LMTLINE safely in CN9K */ 812 sq->lmt_addr = (void *)(nix->lmt_base + 813 ((qid & RVU_CN9K_LMT_SLOT_MASK) << 12)); 814 } 815 816 return rc; 817 nomem: 818 plt_free(sq->fc); 819 fail: 820 return rc; 821 } 822 823 int 824 roc_nix_sq_fini(struct roc_nix_sq *sq) 825 { 826 struct nix *nix; 827 struct mbox *mbox; 828 struct ndc_sync_op *ndc_req; 829 uint16_t qid; 830 int rc = 0; 831 832 if (sq == NULL) 833 return NIX_ERR_PARAM; 834 835 nix = roc_nix_to_nix_priv(sq->roc_nix); 836 mbox = (&nix->dev)->mbox; 837 838 qid = sq->qid; 839 840 rc = nix_tm_sq_flush_pre(sq); 841 842 /* Release SQ context */ 843 if (roc_model_is_cn9k()) 844 rc |= sq_cn9k_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 845 else 846 rc |= sq_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 847 848 /* Sync NDC-NIX-TX for LF */ 849 ndc_req = mbox_alloc_msg_ndc_sync_op(mbox); 850 if (ndc_req == NULL) 851 return -ENOSPC; 852 ndc_req->nix_lf_tx_sync = 1; 853 if (mbox_process(mbox)) 854 rc |= NIX_ERR_NDC_SYNC; 855 856 rc |= nix_tm_sq_flush_post(sq); 857 rc |= roc_npa_pool_destroy(sq->aura_handle); 858 plt_free(sq->fc); 859 plt_free(sq->sqe_mem); 860 nix->sqs[qid] = NULL; 861 862 return rc; 863 } 864