1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <math.h> 6 7 #include "roc_api.h" 8 #include "roc_priv.h" 9 10 static inline uint32_t 11 nix_qsize_to_val(enum nix_q_size qsize) 12 { 13 return (16UL << (qsize * 2)); 14 } 15 16 static inline enum nix_q_size 17 nix_qsize_clampup(uint32_t val) 18 { 19 int i = nix_q_size_16; 20 21 for (; i < nix_q_size_max; i++) 22 if (val <= nix_qsize_to_val(i)) 23 break; 24 25 if (i >= nix_q_size_max) 26 i = nix_q_size_max - 1; 27 28 return i; 29 } 30 31 int 32 roc_nix_rq_ena_dis(struct roc_nix_rq *rq, bool enable) 33 { 34 struct nix *nix = roc_nix_to_nix_priv(rq->roc_nix); 35 struct mbox *mbox = (&nix->dev)->mbox; 36 int rc; 37 38 /* Pkts will be dropped silently if RQ is disabled */ 39 if (roc_model_is_cn9k()) { 40 struct nix_aq_enq_req *aq; 41 42 aq = mbox_alloc_msg_nix_aq_enq(mbox); 43 aq->qidx = rq->qid; 44 aq->ctype = NIX_AQ_CTYPE_RQ; 45 aq->op = NIX_AQ_INSTOP_WRITE; 46 47 aq->rq.ena = enable; 48 aq->rq_mask.ena = ~(aq->rq_mask.ena); 49 } else { 50 struct nix_cn10k_aq_enq_req *aq; 51 52 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 53 aq->qidx = rq->qid; 54 aq->ctype = NIX_AQ_CTYPE_RQ; 55 aq->op = NIX_AQ_INSTOP_WRITE; 56 57 aq->rq.ena = enable; 58 aq->rq_mask.ena = ~(aq->rq_mask.ena); 59 } 60 61 rc = mbox_process(mbox); 62 63 if (roc_model_is_cn10k()) 64 plt_write64(rq->qid, nix->base + NIX_LF_OP_VWQE_FLUSH); 65 return rc; 66 } 67 68 static int 69 rq_cn9k_cfg(struct nix *nix, struct roc_nix_rq *rq, bool cfg, bool ena) 70 { 71 struct mbox *mbox = (&nix->dev)->mbox; 72 struct nix_aq_enq_req *aq; 73 74 aq = mbox_alloc_msg_nix_aq_enq(mbox); 75 aq->qidx = rq->qid; 76 aq->ctype = NIX_AQ_CTYPE_RQ; 77 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 78 79 if (rq->sso_ena) { 80 /* SSO mode */ 81 aq->rq.sso_ena = 1; 82 aq->rq.sso_tt = rq->tt; 83 aq->rq.sso_grp = rq->hwgrp; 84 aq->rq.ena_wqwd = 1; 85 aq->rq.wqe_skip = rq->wqe_skip; 86 aq->rq.wqe_caching = 1; 87 88 aq->rq.good_utag = rq->tag_mask >> 24; 89 aq->rq.bad_utag = rq->tag_mask >> 24; 90 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 91 } else { 92 /* CQ mode */ 93 aq->rq.sso_ena = 0; 94 aq->rq.good_utag = rq->tag_mask >> 24; 95 aq->rq.bad_utag = rq->tag_mask >> 24; 96 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 97 aq->rq.cq = rq->qid; 98 } 99 100 if (rq->ipsech_ena) 101 aq->rq.ipsech_ena = 1; 102 103 aq->rq.spb_ena = 0; 104 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 105 106 /* Sizes must be aligned to 8 bytes */ 107 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 108 return -EINVAL; 109 110 /* Expressed in number of dwords */ 111 aq->rq.first_skip = rq->first_skip / 8; 112 aq->rq.later_skip = rq->later_skip / 8; 113 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 114 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 115 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 116 aq->rq.ena = ena; 117 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 118 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 119 aq->rq.rq_int_ena = 0; 120 /* Many to one reduction */ 121 aq->rq.qint_idx = rq->qid % nix->qints; 122 aq->rq.xqe_drop_ena = 1; 123 124 /* If RED enabled, then fill enable for all cases */ 125 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 126 aq->rq.spb_aura_pass = rq->spb_red_pass; 127 aq->rq.lpb_aura_pass = rq->red_pass; 128 129 aq->rq.spb_aura_drop = rq->spb_red_drop; 130 aq->rq.lpb_aura_drop = rq->red_drop; 131 } 132 133 if (cfg) { 134 if (rq->sso_ena) { 135 /* SSO mode */ 136 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 137 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 138 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 139 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 140 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 141 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 142 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 143 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 144 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 145 } else { 146 /* CQ mode */ 147 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 148 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 149 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 150 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 151 aq->rq_mask.cq = ~aq->rq_mask.cq; 152 } 153 154 if (rq->ipsech_ena) 155 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 156 157 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 158 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 159 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 160 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 161 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 162 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 163 aq->rq_mask.ena = ~aq->rq_mask.ena; 164 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 165 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 166 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 167 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 168 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 169 170 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 171 aq->rq_mask.spb_aura_pass = ~aq->rq_mask.spb_aura_pass; 172 aq->rq_mask.lpb_aura_pass = ~aq->rq_mask.lpb_aura_pass; 173 174 aq->rq_mask.spb_aura_drop = ~aq->rq_mask.spb_aura_drop; 175 aq->rq_mask.lpb_aura_drop = ~aq->rq_mask.lpb_aura_drop; 176 } 177 } 178 179 return 0; 180 } 181 182 static int 183 rq_cfg(struct nix *nix, struct roc_nix_rq *rq, bool cfg, bool ena) 184 { 185 struct mbox *mbox = (&nix->dev)->mbox; 186 struct nix_cn10k_aq_enq_req *aq; 187 188 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 189 aq->qidx = rq->qid; 190 aq->ctype = NIX_AQ_CTYPE_RQ; 191 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 192 193 if (rq->sso_ena) { 194 /* SSO mode */ 195 aq->rq.sso_ena = 1; 196 aq->rq.sso_tt = rq->tt; 197 aq->rq.sso_grp = rq->hwgrp; 198 aq->rq.ena_wqwd = 1; 199 aq->rq.wqe_skip = rq->wqe_skip; 200 aq->rq.wqe_caching = 1; 201 202 aq->rq.good_utag = rq->tag_mask >> 24; 203 aq->rq.bad_utag = rq->tag_mask >> 24; 204 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 205 206 if (rq->vwqe_ena) { 207 aq->rq.vwqe_ena = true; 208 aq->rq.vwqe_skip = rq->vwqe_first_skip; 209 /* Maximal Vector size is (2^(MAX_VSIZE_EXP+2)) */ 210 aq->rq.max_vsize_exp = rq->vwqe_max_sz_exp - 2; 211 aq->rq.vtime_wait = rq->vwqe_wait_tmo; 212 aq->rq.wqe_aura = rq->vwqe_aura_handle; 213 } 214 } else { 215 /* CQ mode */ 216 aq->rq.sso_ena = 0; 217 aq->rq.good_utag = rq->tag_mask >> 24; 218 aq->rq.bad_utag = rq->tag_mask >> 24; 219 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 220 aq->rq.cq = rq->qid; 221 } 222 223 if (rq->ipsech_ena) 224 aq->rq.ipsech_ena = 1; 225 226 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 227 228 /* Sizes must be aligned to 8 bytes */ 229 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 230 return -EINVAL; 231 232 /* Expressed in number of dwords */ 233 aq->rq.first_skip = rq->first_skip / 8; 234 aq->rq.later_skip = rq->later_skip / 8; 235 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 236 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 237 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 238 aq->rq.ena = ena; 239 240 if (rq->spb_ena) { 241 uint32_t spb_sizem1; 242 243 aq->rq.spb_ena = 1; 244 aq->rq.spb_aura = 245 roc_npa_aura_handle_to_aura(rq->spb_aura_handle); 246 247 if (rq->spb_size & 0x7 || 248 rq->spb_size > NIX_RQ_CN10K_SPB_MAX_SIZE) 249 return -EINVAL; 250 251 spb_sizem1 = rq->spb_size / 8; /* Expressed in no. of dwords */ 252 spb_sizem1 -= 1; /* Expressed in size minus one */ 253 aq->rq.spb_sizem1 = spb_sizem1 & 0x3F; 254 aq->rq.spb_high_sizem1 = (spb_sizem1 >> 6) & 0x7; 255 } else { 256 aq->rq.spb_ena = 0; 257 } 258 259 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 260 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 261 aq->rq.rq_int_ena = 0; 262 /* Many to one reduction */ 263 aq->rq.qint_idx = rq->qid % nix->qints; 264 aq->rq.xqe_drop_ena = 1; 265 266 /* If RED enabled, then fill enable for all cases */ 267 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 268 aq->rq.spb_pool_pass = rq->red_pass; 269 aq->rq.spb_aura_pass = rq->red_pass; 270 aq->rq.lpb_pool_pass = rq->red_pass; 271 aq->rq.lpb_aura_pass = rq->red_pass; 272 aq->rq.wqe_pool_pass = rq->red_pass; 273 aq->rq.xqe_pass = rq->red_pass; 274 275 aq->rq.spb_pool_drop = rq->red_drop; 276 aq->rq.spb_aura_drop = rq->red_drop; 277 aq->rq.lpb_pool_drop = rq->red_drop; 278 aq->rq.lpb_aura_drop = rq->red_drop; 279 aq->rq.wqe_pool_drop = rq->red_drop; 280 aq->rq.xqe_drop = rq->red_drop; 281 } 282 283 if (cfg) { 284 if (rq->sso_ena) { 285 /* SSO mode */ 286 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 287 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 288 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 289 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 290 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 291 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 292 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 293 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 294 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 295 if (rq->vwqe_ena) { 296 aq->rq_mask.vwqe_ena = ~aq->rq_mask.vwqe_ena; 297 aq->rq_mask.vwqe_skip = ~aq->rq_mask.vwqe_skip; 298 aq->rq_mask.max_vsize_exp = 299 ~aq->rq_mask.max_vsize_exp; 300 aq->rq_mask.vtime_wait = 301 ~aq->rq_mask.vtime_wait; 302 aq->rq_mask.wqe_aura = ~aq->rq_mask.wqe_aura; 303 } 304 } else { 305 /* CQ mode */ 306 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 307 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 308 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 309 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 310 aq->rq_mask.cq = ~aq->rq_mask.cq; 311 } 312 313 if (rq->ipsech_ena) 314 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 315 316 if (rq->spb_ena) { 317 aq->rq_mask.spb_aura = ~aq->rq_mask.spb_aura; 318 aq->rq_mask.spb_sizem1 = ~aq->rq_mask.spb_sizem1; 319 aq->rq_mask.spb_high_sizem1 = 320 ~aq->rq_mask.spb_high_sizem1; 321 } 322 323 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 324 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 325 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 326 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 327 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 328 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 329 aq->rq_mask.ena = ~aq->rq_mask.ena; 330 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 331 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 332 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 333 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 334 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 335 336 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 337 aq->rq_mask.spb_pool_pass = ~aq->rq_mask.spb_pool_pass; 338 aq->rq_mask.spb_aura_pass = ~aq->rq_mask.spb_aura_pass; 339 aq->rq_mask.lpb_pool_pass = ~aq->rq_mask.lpb_pool_pass; 340 aq->rq_mask.lpb_aura_pass = ~aq->rq_mask.lpb_aura_pass; 341 aq->rq_mask.wqe_pool_pass = ~aq->rq_mask.wqe_pool_pass; 342 aq->rq_mask.xqe_pass = ~aq->rq_mask.xqe_pass; 343 344 aq->rq_mask.spb_pool_drop = ~aq->rq_mask.spb_pool_drop; 345 aq->rq_mask.spb_aura_drop = ~aq->rq_mask.spb_aura_drop; 346 aq->rq_mask.lpb_pool_drop = ~aq->rq_mask.lpb_pool_drop; 347 aq->rq_mask.lpb_aura_drop = ~aq->rq_mask.lpb_aura_drop; 348 aq->rq_mask.wqe_pool_drop = ~aq->rq_mask.wqe_pool_drop; 349 aq->rq_mask.xqe_drop = ~aq->rq_mask.xqe_drop; 350 } 351 } 352 353 return 0; 354 } 355 356 int 357 roc_nix_rq_init(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 358 { 359 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 360 struct mbox *mbox = (&nix->dev)->mbox; 361 bool is_cn9k = roc_model_is_cn9k(); 362 int rc; 363 364 if (roc_nix == NULL || rq == NULL) 365 return NIX_ERR_PARAM; 366 367 if (rq->qid >= nix->nb_rx_queues) 368 return NIX_ERR_QUEUE_INVALID_RANGE; 369 370 rq->roc_nix = roc_nix; 371 372 if (is_cn9k) 373 rc = rq_cn9k_cfg(nix, rq, false, ena); 374 else 375 rc = rq_cfg(nix, rq, false, ena); 376 377 if (rc) 378 return rc; 379 380 return mbox_process(mbox); 381 } 382 383 int 384 roc_nix_rq_modify(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 385 { 386 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 387 struct mbox *mbox = (&nix->dev)->mbox; 388 bool is_cn9k = roc_model_is_cn9k(); 389 int rc; 390 391 if (roc_nix == NULL || rq == NULL) 392 return NIX_ERR_PARAM; 393 394 if (rq->qid >= nix->nb_rx_queues) 395 return NIX_ERR_QUEUE_INVALID_RANGE; 396 397 rq->roc_nix = roc_nix; 398 399 if (is_cn9k) 400 rc = rq_cn9k_cfg(nix, rq, true, ena); 401 else 402 rc = rq_cfg(nix, rq, true, ena); 403 404 if (rc) 405 return rc; 406 407 return mbox_process(mbox); 408 } 409 410 int 411 roc_nix_rq_fini(struct roc_nix_rq *rq) 412 { 413 /* Disabling RQ is sufficient */ 414 return roc_nix_rq_ena_dis(rq, false); 415 } 416 417 int 418 roc_nix_cq_init(struct roc_nix *roc_nix, struct roc_nix_cq *cq) 419 { 420 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 421 struct mbox *mbox = (&nix->dev)->mbox; 422 volatile struct nix_cq_ctx_s *cq_ctx; 423 enum nix_q_size qsize; 424 size_t desc_sz; 425 int rc; 426 427 if (cq == NULL) 428 return NIX_ERR_PARAM; 429 430 if (cq->qid >= nix->nb_rx_queues) 431 return NIX_ERR_QUEUE_INVALID_RANGE; 432 433 qsize = nix_qsize_clampup(cq->nb_desc); 434 cq->nb_desc = nix_qsize_to_val(qsize); 435 cq->qmask = cq->nb_desc - 1; 436 cq->door = nix->base + NIX_LF_CQ_OP_DOOR; 437 cq->status = (int64_t *)(nix->base + NIX_LF_CQ_OP_STATUS); 438 cq->wdata = (uint64_t)cq->qid << 32; 439 cq->roc_nix = roc_nix; 440 441 /* CQE of W16 */ 442 desc_sz = cq->nb_desc * NIX_CQ_ENTRY_SZ; 443 cq->desc_base = plt_zmalloc(desc_sz, NIX_CQ_ALIGN); 444 if (cq->desc_base == NULL) { 445 rc = NIX_ERR_NO_MEM; 446 goto fail; 447 } 448 449 if (roc_model_is_cn9k()) { 450 struct nix_aq_enq_req *aq; 451 452 aq = mbox_alloc_msg_nix_aq_enq(mbox); 453 aq->qidx = cq->qid; 454 aq->ctype = NIX_AQ_CTYPE_CQ; 455 aq->op = NIX_AQ_INSTOP_INIT; 456 cq_ctx = &aq->cq; 457 } else { 458 struct nix_cn10k_aq_enq_req *aq; 459 460 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 461 aq->qidx = cq->qid; 462 aq->ctype = NIX_AQ_CTYPE_CQ; 463 aq->op = NIX_AQ_INSTOP_INIT; 464 cq_ctx = &aq->cq; 465 } 466 467 cq_ctx->ena = 1; 468 cq_ctx->caching = 1; 469 cq_ctx->qsize = qsize; 470 cq_ctx->base = (uint64_t)cq->desc_base; 471 cq_ctx->avg_level = 0xff; 472 cq_ctx->cq_err_int_ena = BIT(NIX_CQERRINT_CQE_FAULT); 473 cq_ctx->cq_err_int_ena |= BIT(NIX_CQERRINT_DOOR_ERR); 474 475 /* Many to one reduction */ 476 cq_ctx->qint_idx = cq->qid % nix->qints; 477 /* Map CQ0 [RQ0] to CINT0 and so on till max 64 irqs */ 478 cq_ctx->cint_idx = cq->qid; 479 480 if (roc_model_is_cn96_a0() || roc_model_is_cn95_a0()) { 481 const float rx_cq_skid = NIX_CQ_FULL_ERRATA_SKID; 482 uint16_t min_rx_drop; 483 484 min_rx_drop = ceil(rx_cq_skid / (float)cq->nb_desc); 485 cq_ctx->drop = min_rx_drop; 486 cq_ctx->drop_ena = 1; 487 cq->drop_thresh = min_rx_drop; 488 } else { 489 cq->drop_thresh = NIX_CQ_THRESH_LEVEL; 490 cq_ctx->drop = cq->drop_thresh; 491 cq_ctx->drop_ena = 1; 492 } 493 494 /* TX pause frames enable flow ctrl on RX side */ 495 if (nix->tx_pause) { 496 /* Single BPID is allocated for all rx channels for now */ 497 cq_ctx->bpid = nix->bpid[0]; 498 cq_ctx->bp = cq_ctx->drop; 499 cq_ctx->bp_ena = 1; 500 } 501 502 rc = mbox_process(mbox); 503 if (rc) 504 goto free_mem; 505 506 return 0; 507 508 free_mem: 509 plt_free(cq->desc_base); 510 fail: 511 return rc; 512 } 513 514 int 515 roc_nix_cq_fini(struct roc_nix_cq *cq) 516 { 517 struct mbox *mbox; 518 struct nix *nix; 519 int rc; 520 521 if (cq == NULL) 522 return NIX_ERR_PARAM; 523 524 nix = roc_nix_to_nix_priv(cq->roc_nix); 525 mbox = (&nix->dev)->mbox; 526 527 /* Disable CQ */ 528 if (roc_model_is_cn9k()) { 529 struct nix_aq_enq_req *aq; 530 531 aq = mbox_alloc_msg_nix_aq_enq(mbox); 532 aq->qidx = cq->qid; 533 aq->ctype = NIX_AQ_CTYPE_CQ; 534 aq->op = NIX_AQ_INSTOP_WRITE; 535 aq->cq.ena = 0; 536 aq->cq.bp_ena = 0; 537 aq->cq_mask.ena = ~aq->cq_mask.ena; 538 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 539 } else { 540 struct nix_cn10k_aq_enq_req *aq; 541 542 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 543 aq->qidx = cq->qid; 544 aq->ctype = NIX_AQ_CTYPE_CQ; 545 aq->op = NIX_AQ_INSTOP_WRITE; 546 aq->cq.ena = 0; 547 aq->cq.bp_ena = 0; 548 aq->cq_mask.ena = ~aq->cq_mask.ena; 549 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 550 } 551 552 rc = mbox_process(mbox); 553 if (rc) 554 return rc; 555 556 plt_free(cq->desc_base); 557 return 0; 558 } 559 560 static int 561 sqb_pool_populate(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 562 { 563 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 564 uint16_t sqes_per_sqb, count, nb_sqb_bufs; 565 struct npa_pool_s pool; 566 struct npa_aura_s aura; 567 uint64_t blk_sz; 568 uint64_t iova; 569 int rc; 570 571 blk_sz = nix->sqb_size; 572 if (sq->max_sqe_sz == roc_nix_maxsqesz_w16) 573 sqes_per_sqb = (blk_sz / 8) / 16; 574 else 575 sqes_per_sqb = (blk_sz / 8) / 8; 576 577 sq->nb_desc = PLT_MAX(256U, sq->nb_desc); 578 nb_sqb_bufs = sq->nb_desc / sqes_per_sqb; 579 nb_sqb_bufs += NIX_SQB_LIST_SPACE; 580 /* Clamp up the SQB count */ 581 nb_sqb_bufs = PLT_MIN(roc_nix->max_sqb_count, 582 (uint16_t)PLT_MAX(NIX_DEF_SQB, nb_sqb_bufs)); 583 584 sq->nb_sqb_bufs = nb_sqb_bufs; 585 sq->sqes_per_sqb_log2 = (uint16_t)plt_log2_u32(sqes_per_sqb); 586 sq->nb_sqb_bufs_adj = 587 nb_sqb_bufs - 588 (PLT_ALIGN_MUL_CEIL(nb_sqb_bufs, sqes_per_sqb) / sqes_per_sqb); 589 sq->nb_sqb_bufs_adj = 590 (sq->nb_sqb_bufs_adj * NIX_SQB_LOWER_THRESH) / 100; 591 592 /* Explicitly set nat_align alone as by default pool is with both 593 * nat_align and buf_offset = 1 which we don't want for SQB. 594 */ 595 memset(&pool, 0, sizeof(struct npa_pool_s)); 596 pool.nat_align = 1; 597 598 memset(&aura, 0, sizeof(aura)); 599 aura.fc_ena = 1; 600 if (roc_model_is_cn9k() || roc_model_is_cn10ka_a0()) 601 aura.fc_stype = 0x0; /* STF */ 602 else 603 aura.fc_stype = 0x3; /* STSTP */ 604 aura.fc_addr = (uint64_t)sq->fc; 605 aura.fc_hyst_bits = 0; /* Store count on all updates */ 606 rc = roc_npa_pool_create(&sq->aura_handle, blk_sz, NIX_MAX_SQB, &aura, 607 &pool); 608 if (rc) 609 goto fail; 610 611 sq->sqe_mem = plt_zmalloc(blk_sz * NIX_MAX_SQB, blk_sz); 612 if (sq->sqe_mem == NULL) { 613 rc = NIX_ERR_NO_MEM; 614 goto nomem; 615 } 616 617 /* Fill the initial buffers */ 618 iova = (uint64_t)sq->sqe_mem; 619 for (count = 0; count < NIX_MAX_SQB; count++) { 620 roc_npa_aura_op_free(sq->aura_handle, 0, iova); 621 iova += blk_sz; 622 } 623 roc_npa_aura_op_range_set(sq->aura_handle, (uint64_t)sq->sqe_mem, iova); 624 roc_npa_aura_limit_modify(sq->aura_handle, sq->nb_sqb_bufs); 625 sq->aura_sqb_bufs = NIX_MAX_SQB; 626 627 return rc; 628 nomem: 629 roc_npa_pool_destroy(sq->aura_handle); 630 fail: 631 return rc; 632 } 633 634 static void 635 sq_cn9k_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 636 uint16_t smq) 637 { 638 struct mbox *mbox = (&nix->dev)->mbox; 639 struct nix_aq_enq_req *aq; 640 641 aq = mbox_alloc_msg_nix_aq_enq(mbox); 642 aq->qidx = sq->qid; 643 aq->ctype = NIX_AQ_CTYPE_SQ; 644 aq->op = NIX_AQ_INSTOP_INIT; 645 aq->sq.max_sqe_size = sq->max_sqe_sz; 646 647 aq->sq.max_sqe_size = sq->max_sqe_sz; 648 aq->sq.smq = smq; 649 aq->sq.smq_rr_quantum = rr_quantum; 650 aq->sq.default_chan = nix->tx_chan_base; 651 aq->sq.sqe_stype = NIX_STYPE_STF; 652 aq->sq.ena = 1; 653 aq->sq.sso_ena = !!sq->sso_ena; 654 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 655 aq->sq.sqe_stype = NIX_STYPE_STP; 656 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 657 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 658 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 659 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 660 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 661 662 /* Many to one reduction */ 663 aq->sq.qint_idx = sq->qid % nix->qints; 664 } 665 666 static int 667 sq_cn9k_fini(struct nix *nix, struct roc_nix_sq *sq) 668 { 669 struct mbox *mbox = (&nix->dev)->mbox; 670 struct nix_aq_enq_rsp *rsp; 671 struct nix_aq_enq_req *aq; 672 uint16_t sqes_per_sqb; 673 void *sqb_buf; 674 int rc, count; 675 676 aq = mbox_alloc_msg_nix_aq_enq(mbox); 677 aq->qidx = sq->qid; 678 aq->ctype = NIX_AQ_CTYPE_SQ; 679 aq->op = NIX_AQ_INSTOP_READ; 680 rc = mbox_process_msg(mbox, (void *)&rsp); 681 if (rc) 682 return rc; 683 684 /* Check if sq is already cleaned up */ 685 if (!rsp->sq.ena) 686 return 0; 687 688 /* Disable sq */ 689 aq = mbox_alloc_msg_nix_aq_enq(mbox); 690 aq->qidx = sq->qid; 691 aq->ctype = NIX_AQ_CTYPE_SQ; 692 aq->op = NIX_AQ_INSTOP_WRITE; 693 aq->sq_mask.ena = ~aq->sq_mask.ena; 694 aq->sq.ena = 0; 695 rc = mbox_process(mbox); 696 if (rc) 697 return rc; 698 699 /* Read SQ and free sqb's */ 700 aq = mbox_alloc_msg_nix_aq_enq(mbox); 701 aq->qidx = sq->qid; 702 aq->ctype = NIX_AQ_CTYPE_SQ; 703 aq->op = NIX_AQ_INSTOP_READ; 704 rc = mbox_process_msg(mbox, (void *)&rsp); 705 if (rc) 706 return rc; 707 708 if (aq->sq.smq_pend) 709 plt_err("SQ has pending SQE's"); 710 711 count = aq->sq.sqb_count; 712 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 713 /* Free SQB's that are used */ 714 sqb_buf = (void *)rsp->sq.head_sqb; 715 while (count) { 716 void *next_sqb; 717 718 next_sqb = *(void **)((uintptr_t)sqb_buf + 719 (uint32_t)((sqes_per_sqb - 1) * 720 sq->max_sqe_sz)); 721 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 722 sqb_buf = next_sqb; 723 count--; 724 } 725 726 /* Free next to use sqb */ 727 if (rsp->sq.next_sqb) 728 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 729 return 0; 730 } 731 732 static void 733 sq_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 734 uint16_t smq) 735 { 736 struct mbox *mbox = (&nix->dev)->mbox; 737 struct nix_cn10k_aq_enq_req *aq; 738 739 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 740 aq->qidx = sq->qid; 741 aq->ctype = NIX_AQ_CTYPE_SQ; 742 aq->op = NIX_AQ_INSTOP_INIT; 743 aq->sq.max_sqe_size = sq->max_sqe_sz; 744 745 aq->sq.max_sqe_size = sq->max_sqe_sz; 746 aq->sq.smq = smq; 747 aq->sq.smq_rr_weight = rr_quantum; 748 aq->sq.default_chan = nix->tx_chan_base; 749 aq->sq.sqe_stype = NIX_STYPE_STF; 750 aq->sq.ena = 1; 751 aq->sq.sso_ena = !!sq->sso_ena; 752 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 753 aq->sq.sqe_stype = NIX_STYPE_STP; 754 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 755 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 756 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 757 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 758 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 759 760 /* Many to one reduction */ 761 aq->sq.qint_idx = sq->qid % nix->qints; 762 } 763 764 static int 765 sq_fini(struct nix *nix, struct roc_nix_sq *sq) 766 { 767 struct mbox *mbox = (&nix->dev)->mbox; 768 struct nix_cn10k_aq_enq_rsp *rsp; 769 struct nix_cn10k_aq_enq_req *aq; 770 uint16_t sqes_per_sqb; 771 void *sqb_buf; 772 int rc, count; 773 774 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 775 aq->qidx = sq->qid; 776 aq->ctype = NIX_AQ_CTYPE_SQ; 777 aq->op = NIX_AQ_INSTOP_READ; 778 rc = mbox_process_msg(mbox, (void *)&rsp); 779 if (rc) 780 return rc; 781 782 /* Check if sq is already cleaned up */ 783 if (!rsp->sq.ena) 784 return 0; 785 786 /* Disable sq */ 787 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 788 aq->qidx = sq->qid; 789 aq->ctype = NIX_AQ_CTYPE_SQ; 790 aq->op = NIX_AQ_INSTOP_WRITE; 791 aq->sq_mask.ena = ~aq->sq_mask.ena; 792 aq->sq.ena = 0; 793 rc = mbox_process(mbox); 794 if (rc) 795 return rc; 796 797 /* Read SQ and free sqb's */ 798 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 799 aq->qidx = sq->qid; 800 aq->ctype = NIX_AQ_CTYPE_SQ; 801 aq->op = NIX_AQ_INSTOP_READ; 802 rc = mbox_process_msg(mbox, (void *)&rsp); 803 if (rc) 804 return rc; 805 806 if (aq->sq.smq_pend) 807 plt_err("SQ has pending SQE's"); 808 809 count = aq->sq.sqb_count; 810 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 811 /* Free SQB's that are used */ 812 sqb_buf = (void *)rsp->sq.head_sqb; 813 while (count) { 814 void *next_sqb; 815 816 next_sqb = *(void **)((uintptr_t)sqb_buf + 817 (uint32_t)((sqes_per_sqb - 1) * 818 sq->max_sqe_sz)); 819 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 820 sqb_buf = next_sqb; 821 count--; 822 } 823 824 /* Free next to use sqb */ 825 if (rsp->sq.next_sqb) 826 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 827 return 0; 828 } 829 830 int 831 roc_nix_sq_init(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 832 { 833 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 834 struct mbox *mbox = (&nix->dev)->mbox; 835 uint16_t qid, smq = UINT16_MAX; 836 uint32_t rr_quantum = 0; 837 int rc; 838 839 if (sq == NULL) 840 return NIX_ERR_PARAM; 841 842 qid = sq->qid; 843 if (qid >= nix->nb_tx_queues) 844 return NIX_ERR_QUEUE_INVALID_RANGE; 845 846 sq->roc_nix = roc_nix; 847 /* 848 * Allocate memory for flow control updates from HW. 849 * Alloc one cache line, so that fits all FC_STYPE modes. 850 */ 851 sq->fc = plt_zmalloc(ROC_ALIGN, ROC_ALIGN); 852 if (sq->fc == NULL) { 853 rc = NIX_ERR_NO_MEM; 854 goto fail; 855 } 856 857 rc = sqb_pool_populate(roc_nix, sq); 858 if (rc) 859 goto nomem; 860 861 rc = nix_tm_leaf_data_get(nix, sq->qid, &rr_quantum, &smq); 862 if (rc) { 863 rc = NIX_ERR_TM_LEAF_NODE_GET; 864 goto nomem; 865 } 866 867 /* Init SQ context */ 868 if (roc_model_is_cn9k()) 869 sq_cn9k_init(nix, sq, rr_quantum, smq); 870 else 871 sq_init(nix, sq, rr_quantum, smq); 872 873 rc = mbox_process(mbox); 874 if (rc) 875 goto nomem; 876 877 nix->sqs[qid] = sq; 878 sq->io_addr = nix->base + NIX_LF_OP_SENDX(0); 879 /* Evenly distribute LMT slot for each sq */ 880 if (roc_model_is_cn9k()) { 881 /* Multiple cores/SQ's can use same LMTLINE safely in CN9K */ 882 sq->lmt_addr = (void *)(nix->lmt_base + 883 ((qid & RVU_CN9K_LMT_SLOT_MASK) << 12)); 884 } 885 886 return rc; 887 nomem: 888 plt_free(sq->fc); 889 fail: 890 return rc; 891 } 892 893 int 894 roc_nix_sq_fini(struct roc_nix_sq *sq) 895 { 896 struct nix *nix; 897 struct mbox *mbox; 898 struct ndc_sync_op *ndc_req; 899 uint16_t qid; 900 int rc = 0; 901 902 if (sq == NULL) 903 return NIX_ERR_PARAM; 904 905 nix = roc_nix_to_nix_priv(sq->roc_nix); 906 mbox = (&nix->dev)->mbox; 907 908 qid = sq->qid; 909 910 rc = nix_tm_sq_flush_pre(sq); 911 912 /* Release SQ context */ 913 if (roc_model_is_cn9k()) 914 rc |= sq_cn9k_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 915 else 916 rc |= sq_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 917 918 /* Sync NDC-NIX-TX for LF */ 919 ndc_req = mbox_alloc_msg_ndc_sync_op(mbox); 920 if (ndc_req == NULL) 921 return -ENOSPC; 922 ndc_req->nix_lf_tx_sync = 1; 923 if (mbox_process(mbox)) 924 rc |= NIX_ERR_NDC_SYNC; 925 926 rc |= nix_tm_sq_flush_post(sq); 927 rc |= roc_npa_pool_destroy(sq->aura_handle); 928 plt_free(sq->fc); 929 plt_free(sq->sqe_mem); 930 nix->sqs[qid] = NULL; 931 932 return rc; 933 } 934