1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
3 */
4
5 #include <math.h>
6
7 #include "roc_api.h"
8 #include "roc_priv.h"
9
10 static inline uint32_t
nix_qsize_to_val(enum nix_q_size qsize)11 nix_qsize_to_val(enum nix_q_size qsize)
12 {
13 return (16UL << (qsize * 2));
14 }
15
16 static inline enum nix_q_size
nix_qsize_clampup(uint32_t val)17 nix_qsize_clampup(uint32_t val)
18 {
19 int i = nix_q_size_16;
20
21 for (; i < nix_q_size_max; i++)
22 if (val <= nix_qsize_to_val(i))
23 break;
24
25 if (i >= nix_q_size_max)
26 i = nix_q_size_max - 1;
27
28 return i;
29 }
30
31 void
nix_rq_vwqe_flush(struct roc_nix_rq * rq,uint16_t vwqe_interval)32 nix_rq_vwqe_flush(struct roc_nix_rq *rq, uint16_t vwqe_interval)
33 {
34 uint64_t wait_ns;
35
36 if (!roc_model_is_cn10k())
37 return;
38 /* Due to HW errata writes to VWQE_FLUSH might hang, so instead
39 * wait for max vwqe timeout interval.
40 */
41 if (rq->vwqe_ena) {
42 wait_ns = rq->vwqe_wait_tmo * (vwqe_interval + 1) * 100;
43 plt_delay_us((wait_ns / 1E3) + 1);
44 }
45 }
46
47 int
nix_rq_ena_dis(struct dev * dev,struct roc_nix_rq * rq,bool enable)48 nix_rq_ena_dis(struct dev *dev, struct roc_nix_rq *rq, bool enable)
49 {
50 struct mbox *mbox = dev->mbox;
51
52 /* Pkts will be dropped silently if RQ is disabled */
53 if (roc_model_is_cn9k()) {
54 struct nix_aq_enq_req *aq;
55
56 aq = mbox_alloc_msg_nix_aq_enq(mbox);
57 if (!aq)
58 return -ENOSPC;
59
60 aq->qidx = rq->qid;
61 aq->ctype = NIX_AQ_CTYPE_RQ;
62 aq->op = NIX_AQ_INSTOP_WRITE;
63
64 aq->rq.ena = enable;
65 aq->rq_mask.ena = ~(aq->rq_mask.ena);
66 } else {
67 struct nix_cn10k_aq_enq_req *aq;
68
69 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox);
70 if (!aq)
71 return -ENOSPC;
72
73 aq->qidx = rq->qid;
74 aq->ctype = NIX_AQ_CTYPE_RQ;
75 aq->op = NIX_AQ_INSTOP_WRITE;
76
77 aq->rq.ena = enable;
78 aq->rq_mask.ena = ~(aq->rq_mask.ena);
79 }
80
81 return mbox_process(mbox);
82 }
83
84 int
roc_nix_rq_ena_dis(struct roc_nix_rq * rq,bool enable)85 roc_nix_rq_ena_dis(struct roc_nix_rq *rq, bool enable)
86 {
87 struct nix *nix = roc_nix_to_nix_priv(rq->roc_nix);
88 int rc;
89
90 rc = nix_rq_ena_dis(&nix->dev, rq, enable);
91 nix_rq_vwqe_flush(rq, nix->vwqe_interval);
92
93 return rc;
94 }
95
96 int
nix_rq_cn9k_cfg(struct dev * dev,struct roc_nix_rq * rq,uint16_t qints,bool cfg,bool ena)97 nix_rq_cn9k_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints,
98 bool cfg, bool ena)
99 {
100 struct mbox *mbox = dev->mbox;
101 struct nix_aq_enq_req *aq;
102
103 aq = mbox_alloc_msg_nix_aq_enq(mbox);
104 if (!aq)
105 return -ENOSPC;
106
107 aq->qidx = rq->qid;
108 aq->ctype = NIX_AQ_CTYPE_RQ;
109 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT;
110
111 if (rq->sso_ena) {
112 /* SSO mode */
113 aq->rq.sso_ena = 1;
114 aq->rq.sso_tt = rq->tt;
115 aq->rq.sso_grp = rq->hwgrp;
116 aq->rq.ena_wqwd = 1;
117 aq->rq.wqe_skip = rq->wqe_skip;
118 aq->rq.wqe_caching = 1;
119
120 aq->rq.good_utag = rq->tag_mask >> 24;
121 aq->rq.bad_utag = rq->tag_mask >> 24;
122 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0);
123 } else {
124 /* CQ mode */
125 aq->rq.sso_ena = 0;
126 aq->rq.good_utag = rq->tag_mask >> 24;
127 aq->rq.bad_utag = rq->tag_mask >> 24;
128 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0);
129 aq->rq.cq = rq->qid;
130 }
131
132 if (rq->ipsech_ena)
133 aq->rq.ipsech_ena = 1;
134
135 aq->rq.spb_ena = 0;
136 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle);
137
138 /* Sizes must be aligned to 8 bytes */
139 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7)
140 return -EINVAL;
141
142 /* Expressed in number of dwords */
143 aq->rq.first_skip = rq->first_skip / 8;
144 aq->rq.later_skip = rq->later_skip / 8;
145 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */
146 aq->rq.lpb_sizem1 = rq->lpb_size / 8;
147 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */
148 aq->rq.ena = ena;
149 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */
150 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */
151 aq->rq.rq_int_ena = 0;
152 /* Many to one reduction */
153 aq->rq.qint_idx = rq->qid % qints;
154 aq->rq.xqe_drop_ena = 1;
155
156 /* If RED enabled, then fill enable for all cases */
157 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) {
158 aq->rq.spb_pool_pass = rq->spb_red_pass;
159 aq->rq.lpb_pool_pass = rq->red_pass;
160
161 aq->rq.spb_pool_drop = rq->spb_red_drop;
162 aq->rq.lpb_pool_drop = rq->red_drop;
163 }
164
165 if (cfg) {
166 if (rq->sso_ena) {
167 /* SSO mode */
168 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena;
169 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt;
170 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp;
171 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd;
172 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip;
173 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching;
174 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag;
175 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag;
176 aq->rq_mask.ltag = ~aq->rq_mask.ltag;
177 } else {
178 /* CQ mode */
179 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena;
180 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag;
181 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag;
182 aq->rq_mask.ltag = ~aq->rq_mask.ltag;
183 aq->rq_mask.cq = ~aq->rq_mask.cq;
184 }
185
186 if (rq->ipsech_ena)
187 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena;
188
189 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena;
190 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura;
191 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip;
192 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip;
193 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw;
194 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1;
195 aq->rq_mask.ena = ~aq->rq_mask.ena;
196 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching;
197 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size;
198 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena;
199 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx;
200 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena;
201
202 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) {
203 aq->rq_mask.spb_pool_pass = ~aq->rq_mask.spb_pool_pass;
204 aq->rq_mask.lpb_pool_pass = ~aq->rq_mask.lpb_pool_pass;
205
206 aq->rq_mask.spb_pool_drop = ~aq->rq_mask.spb_pool_drop;
207 aq->rq_mask.lpb_pool_drop = ~aq->rq_mask.lpb_pool_drop;
208 }
209 }
210
211 return 0;
212 }
213
214 int
nix_rq_cfg(struct dev * dev,struct roc_nix_rq * rq,uint16_t qints,bool cfg,bool ena)215 nix_rq_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, bool cfg,
216 bool ena)
217 {
218 struct nix_cn10k_aq_enq_req *aq;
219 struct mbox *mbox = dev->mbox;
220
221 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox);
222 if (!aq)
223 return -ENOSPC;
224
225 aq->qidx = rq->qid;
226 aq->ctype = NIX_AQ_CTYPE_RQ;
227 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT;
228
229 if (rq->sso_ena) {
230 /* SSO mode */
231 aq->rq.sso_ena = 1;
232 aq->rq.sso_tt = rq->tt;
233 aq->rq.sso_grp = rq->hwgrp;
234 aq->rq.ena_wqwd = 1;
235 aq->rq.wqe_skip = rq->wqe_skip;
236 aq->rq.wqe_caching = 1;
237
238 aq->rq.good_utag = rq->tag_mask >> 24;
239 aq->rq.bad_utag = rq->tag_mask >> 24;
240 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0);
241
242 if (rq->vwqe_ena) {
243 aq->rq.vwqe_ena = true;
244 aq->rq.vwqe_skip = rq->vwqe_first_skip;
245 /* Maximal Vector size is (2^(MAX_VSIZE_EXP+2)) */
246 aq->rq.max_vsize_exp = rq->vwqe_max_sz_exp - 2;
247 aq->rq.vtime_wait = rq->vwqe_wait_tmo;
248 aq->rq.wqe_aura = rq->vwqe_aura_handle;
249 }
250 } else {
251 /* CQ mode */
252 aq->rq.sso_ena = 0;
253 aq->rq.good_utag = rq->tag_mask >> 24;
254 aq->rq.bad_utag = rq->tag_mask >> 24;
255 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0);
256 aq->rq.cq = rq->qid;
257 }
258
259 if (rq->ipsech_ena) {
260 aq->rq.ipsech_ena = 1;
261 aq->rq.ipsecd_drop_en = 1;
262 }
263
264 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle);
265
266 /* Sizes must be aligned to 8 bytes */
267 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7)
268 return -EINVAL;
269
270 /* Expressed in number of dwords */
271 aq->rq.first_skip = rq->first_skip / 8;
272 aq->rq.later_skip = rq->later_skip / 8;
273 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */
274 aq->rq.lpb_sizem1 = rq->lpb_size / 8;
275 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */
276 aq->rq.ena = ena;
277
278 if (rq->spb_ena) {
279 uint32_t spb_sizem1;
280
281 aq->rq.spb_ena = 1;
282 aq->rq.spb_aura =
283 roc_npa_aura_handle_to_aura(rq->spb_aura_handle);
284
285 if (rq->spb_size & 0x7 ||
286 rq->spb_size > NIX_RQ_CN10K_SPB_MAX_SIZE)
287 return -EINVAL;
288
289 spb_sizem1 = rq->spb_size / 8; /* Expressed in no. of dwords */
290 spb_sizem1 -= 1; /* Expressed in size minus one */
291 aq->rq.spb_sizem1 = spb_sizem1 & 0x3F;
292 aq->rq.spb_high_sizem1 = (spb_sizem1 >> 6) & 0x7;
293 } else {
294 aq->rq.spb_ena = 0;
295 }
296
297 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */
298 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */
299 aq->rq.rq_int_ena = 0;
300 /* Many to one reduction */
301 aq->rq.qint_idx = rq->qid % qints;
302 aq->rq.xqe_drop_ena = 0;
303 aq->rq.lpb_drop_ena = rq->lpb_drop_ena;
304 aq->rq.spb_drop_ena = rq->spb_drop_ena;
305
306 /* If RED enabled, then fill enable for all cases */
307 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) {
308 aq->rq.spb_pool_pass = rq->spb_red_pass;
309 aq->rq.lpb_pool_pass = rq->red_pass;
310 aq->rq.wqe_pool_pass = rq->red_pass;
311 aq->rq.xqe_pass = rq->red_pass;
312
313 aq->rq.spb_pool_drop = rq->spb_red_drop;
314 aq->rq.lpb_pool_drop = rq->red_drop;
315 aq->rq.wqe_pool_drop = rq->red_drop;
316 aq->rq.xqe_drop = rq->red_drop;
317 }
318
319 if (cfg) {
320 if (rq->sso_ena) {
321 /* SSO mode */
322 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena;
323 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt;
324 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp;
325 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd;
326 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip;
327 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching;
328 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag;
329 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag;
330 aq->rq_mask.ltag = ~aq->rq_mask.ltag;
331 if (rq->vwqe_ena) {
332 aq->rq_mask.vwqe_ena = ~aq->rq_mask.vwqe_ena;
333 aq->rq_mask.vwqe_skip = ~aq->rq_mask.vwqe_skip;
334 aq->rq_mask.max_vsize_exp =
335 ~aq->rq_mask.max_vsize_exp;
336 aq->rq_mask.vtime_wait =
337 ~aq->rq_mask.vtime_wait;
338 aq->rq_mask.wqe_aura = ~aq->rq_mask.wqe_aura;
339 }
340 } else {
341 /* CQ mode */
342 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena;
343 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag;
344 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag;
345 aq->rq_mask.ltag = ~aq->rq_mask.ltag;
346 aq->rq_mask.cq = ~aq->rq_mask.cq;
347 }
348
349 if (rq->ipsech_ena)
350 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena;
351
352 if (rq->spb_ena) {
353 aq->rq_mask.spb_aura = ~aq->rq_mask.spb_aura;
354 aq->rq_mask.spb_sizem1 = ~aq->rq_mask.spb_sizem1;
355 aq->rq_mask.spb_high_sizem1 =
356 ~aq->rq_mask.spb_high_sizem1;
357 }
358
359 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena;
360 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura;
361 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip;
362 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip;
363 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw;
364 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1;
365 aq->rq_mask.ena = ~aq->rq_mask.ena;
366 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching;
367 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size;
368 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena;
369 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx;
370 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena;
371 aq->rq_mask.lpb_drop_ena = ~aq->rq_mask.lpb_drop_ena;
372 aq->rq_mask.spb_drop_ena = ~aq->rq_mask.spb_drop_ena;
373
374 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) {
375 aq->rq_mask.spb_pool_pass = ~aq->rq_mask.spb_pool_pass;
376 aq->rq_mask.lpb_pool_pass = ~aq->rq_mask.lpb_pool_pass;
377 aq->rq_mask.wqe_pool_pass = ~aq->rq_mask.wqe_pool_pass;
378 aq->rq_mask.xqe_pass = ~aq->rq_mask.xqe_pass;
379
380 aq->rq_mask.spb_pool_drop = ~aq->rq_mask.spb_pool_drop;
381 aq->rq_mask.lpb_pool_drop = ~aq->rq_mask.lpb_pool_drop;
382 aq->rq_mask.wqe_pool_drop = ~aq->rq_mask.wqe_pool_drop;
383 aq->rq_mask.xqe_drop = ~aq->rq_mask.xqe_drop;
384 }
385 }
386
387 return 0;
388 }
389
390 int
roc_nix_rq_init(struct roc_nix * roc_nix,struct roc_nix_rq * rq,bool ena)391 roc_nix_rq_init(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena)
392 {
393 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
394 struct mbox *mbox = (&nix->dev)->mbox;
395 bool is_cn9k = roc_model_is_cn9k();
396 struct dev *dev = &nix->dev;
397 int rc;
398
399 if (roc_nix == NULL || rq == NULL)
400 return NIX_ERR_PARAM;
401
402 if (rq->qid >= nix->nb_rx_queues)
403 return NIX_ERR_QUEUE_INVALID_RANGE;
404
405 rq->roc_nix = roc_nix;
406
407 if (is_cn9k)
408 rc = nix_rq_cn9k_cfg(dev, rq, nix->qints, false, ena);
409 else
410 rc = nix_rq_cfg(dev, rq, nix->qints, false, ena);
411
412 if (rc)
413 return rc;
414
415 rc = mbox_process(mbox);
416 if (rc)
417 return rc;
418
419 return nix_tel_node_add_rq(rq);
420 }
421
422 int
roc_nix_rq_modify(struct roc_nix * roc_nix,struct roc_nix_rq * rq,bool ena)423 roc_nix_rq_modify(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena)
424 {
425 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
426 struct mbox *mbox = (&nix->dev)->mbox;
427 bool is_cn9k = roc_model_is_cn9k();
428 struct dev *dev = &nix->dev;
429 int rc;
430
431 if (roc_nix == NULL || rq == NULL)
432 return NIX_ERR_PARAM;
433
434 if (rq->qid >= nix->nb_rx_queues)
435 return NIX_ERR_QUEUE_INVALID_RANGE;
436
437 rq->roc_nix = roc_nix;
438
439 if (is_cn9k)
440 rc = nix_rq_cn9k_cfg(dev, rq, nix->qints, true, ena);
441 else
442 rc = nix_rq_cfg(dev, rq, nix->qints, true, ena);
443
444 if (rc)
445 return rc;
446
447 rc = mbox_process(mbox);
448 if (rc)
449 return rc;
450
451 return nix_tel_node_add_rq(rq);
452 }
453
454 int
roc_nix_rq_fini(struct roc_nix_rq * rq)455 roc_nix_rq_fini(struct roc_nix_rq *rq)
456 {
457 /* Disabling RQ is sufficient */
458 return roc_nix_rq_ena_dis(rq, false);
459 }
460
461 int
roc_nix_cq_init(struct roc_nix * roc_nix,struct roc_nix_cq * cq)462 roc_nix_cq_init(struct roc_nix *roc_nix, struct roc_nix_cq *cq)
463 {
464 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
465 struct mbox *mbox = (&nix->dev)->mbox;
466 volatile struct nix_cq_ctx_s *cq_ctx;
467 enum nix_q_size qsize;
468 size_t desc_sz;
469 int rc;
470
471 if (cq == NULL)
472 return NIX_ERR_PARAM;
473
474 if (cq->qid >= nix->nb_rx_queues)
475 return NIX_ERR_QUEUE_INVALID_RANGE;
476
477 qsize = nix_qsize_clampup(cq->nb_desc);
478 cq->nb_desc = nix_qsize_to_val(qsize);
479 cq->qmask = cq->nb_desc - 1;
480 cq->door = nix->base + NIX_LF_CQ_OP_DOOR;
481 cq->status = (int64_t *)(nix->base + NIX_LF_CQ_OP_STATUS);
482 cq->wdata = (uint64_t)cq->qid << 32;
483 cq->roc_nix = roc_nix;
484
485 /* CQE of W16 */
486 desc_sz = cq->nb_desc * NIX_CQ_ENTRY_SZ;
487 cq->desc_base = plt_zmalloc(desc_sz, NIX_CQ_ALIGN);
488 if (cq->desc_base == NULL) {
489 rc = NIX_ERR_NO_MEM;
490 goto fail;
491 }
492
493 if (roc_model_is_cn9k()) {
494 struct nix_aq_enq_req *aq;
495
496 aq = mbox_alloc_msg_nix_aq_enq(mbox);
497 if (!aq)
498 return -ENOSPC;
499
500 aq->qidx = cq->qid;
501 aq->ctype = NIX_AQ_CTYPE_CQ;
502 aq->op = NIX_AQ_INSTOP_INIT;
503 cq_ctx = &aq->cq;
504 } else {
505 struct nix_cn10k_aq_enq_req *aq;
506
507 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox);
508 if (!aq)
509 return -ENOSPC;
510
511 aq->qidx = cq->qid;
512 aq->ctype = NIX_AQ_CTYPE_CQ;
513 aq->op = NIX_AQ_INSTOP_INIT;
514 cq_ctx = &aq->cq;
515 }
516
517 cq_ctx->ena = 1;
518 cq_ctx->caching = 1;
519 cq_ctx->qsize = qsize;
520 cq_ctx->base = (uint64_t)cq->desc_base;
521 cq_ctx->avg_level = 0xff;
522 cq_ctx->cq_err_int_ena = BIT(NIX_CQERRINT_CQE_FAULT);
523 cq_ctx->cq_err_int_ena |= BIT(NIX_CQERRINT_DOOR_ERR);
524
525 /* Many to one reduction */
526 cq_ctx->qint_idx = cq->qid % nix->qints;
527 /* Map CQ0 [RQ0] to CINT0 and so on till max 64 irqs */
528 cq_ctx->cint_idx = cq->qid;
529
530 if (roc_errata_nix_has_cq_min_size_4k()) {
531 const float rx_cq_skid = NIX_CQ_FULL_ERRATA_SKID;
532 uint16_t min_rx_drop;
533
534 min_rx_drop = ceil(rx_cq_skid / (float)cq->nb_desc);
535 cq_ctx->drop = min_rx_drop;
536 cq_ctx->drop_ena = 1;
537 cq->drop_thresh = min_rx_drop;
538 } else {
539 cq->drop_thresh = NIX_CQ_THRESH_LEVEL;
540 /* Drop processing or red drop cannot be enabled due to
541 * due to packets coming for second pass from CPT.
542 */
543 if (!roc_nix_inl_inb_is_enabled(roc_nix)) {
544 cq_ctx->drop = cq->drop_thresh;
545 cq_ctx->drop_ena = 1;
546 }
547 }
548
549 /* TX pause frames enable flow ctrl on RX side */
550 if (nix->tx_pause) {
551 /* Single BPID is allocated for all rx channels for now */
552 cq_ctx->bpid = nix->bpid[0];
553 cq_ctx->bp = cq->drop_thresh;
554 cq_ctx->bp_ena = 1;
555 }
556
557 rc = mbox_process(mbox);
558 if (rc)
559 goto free_mem;
560
561 return nix_tel_node_add_cq(cq);
562
563 free_mem:
564 plt_free(cq->desc_base);
565 fail:
566 return rc;
567 }
568
569 int
roc_nix_cq_fini(struct roc_nix_cq * cq)570 roc_nix_cq_fini(struct roc_nix_cq *cq)
571 {
572 struct mbox *mbox;
573 struct nix *nix;
574 int rc;
575
576 if (cq == NULL)
577 return NIX_ERR_PARAM;
578
579 nix = roc_nix_to_nix_priv(cq->roc_nix);
580 mbox = (&nix->dev)->mbox;
581
582 /* Disable CQ */
583 if (roc_model_is_cn9k()) {
584 struct nix_aq_enq_req *aq;
585
586 aq = mbox_alloc_msg_nix_aq_enq(mbox);
587 if (!aq)
588 return -ENOSPC;
589
590 aq->qidx = cq->qid;
591 aq->ctype = NIX_AQ_CTYPE_CQ;
592 aq->op = NIX_AQ_INSTOP_WRITE;
593 aq->cq.ena = 0;
594 aq->cq.bp_ena = 0;
595 aq->cq_mask.ena = ~aq->cq_mask.ena;
596 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena;
597 } else {
598 struct nix_cn10k_aq_enq_req *aq;
599
600 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox);
601 if (!aq)
602 return -ENOSPC;
603
604 aq->qidx = cq->qid;
605 aq->ctype = NIX_AQ_CTYPE_CQ;
606 aq->op = NIX_AQ_INSTOP_WRITE;
607 aq->cq.ena = 0;
608 aq->cq.bp_ena = 0;
609 aq->cq_mask.ena = ~aq->cq_mask.ena;
610 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena;
611 }
612
613 rc = mbox_process(mbox);
614 if (rc)
615 return rc;
616
617 plt_free(cq->desc_base);
618 return 0;
619 }
620
621 static int
sqb_pool_populate(struct roc_nix * roc_nix,struct roc_nix_sq * sq)622 sqb_pool_populate(struct roc_nix *roc_nix, struct roc_nix_sq *sq)
623 {
624 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
625 uint16_t sqes_per_sqb, count, nb_sqb_bufs;
626 struct npa_pool_s pool;
627 struct npa_aura_s aura;
628 uint64_t blk_sz;
629 uint64_t iova;
630 int rc;
631
632 blk_sz = nix->sqb_size;
633 if (sq->max_sqe_sz == roc_nix_maxsqesz_w16)
634 sqes_per_sqb = (blk_sz / 8) / 16;
635 else
636 sqes_per_sqb = (blk_sz / 8) / 8;
637
638 sq->nb_desc = PLT_MAX(256U, sq->nb_desc);
639 nb_sqb_bufs = sq->nb_desc / sqes_per_sqb;
640 nb_sqb_bufs += NIX_SQB_LIST_SPACE;
641 /* Clamp up the SQB count */
642 nb_sqb_bufs = PLT_MIN(roc_nix->max_sqb_count,
643 (uint16_t)PLT_MAX(NIX_DEF_SQB, nb_sqb_bufs));
644
645 sq->nb_sqb_bufs = nb_sqb_bufs;
646 sq->sqes_per_sqb_log2 = (uint16_t)plt_log2_u32(sqes_per_sqb);
647 sq->nb_sqb_bufs_adj =
648 nb_sqb_bufs -
649 (PLT_ALIGN_MUL_CEIL(nb_sqb_bufs, sqes_per_sqb) / sqes_per_sqb);
650 sq->nb_sqb_bufs_adj =
651 (sq->nb_sqb_bufs_adj * NIX_SQB_LOWER_THRESH) / 100;
652
653 /* Explicitly set nat_align alone as by default pool is with both
654 * nat_align and buf_offset = 1 which we don't want for SQB.
655 */
656 memset(&pool, 0, sizeof(struct npa_pool_s));
657 pool.nat_align = 1;
658
659 memset(&aura, 0, sizeof(aura));
660 aura.fc_ena = 1;
661 if (roc_model_is_cn9k() || roc_errata_npa_has_no_fc_stype_ststp())
662 aura.fc_stype = 0x0; /* STF */
663 else
664 aura.fc_stype = 0x3; /* STSTP */
665 aura.fc_addr = (uint64_t)sq->fc;
666 aura.fc_hyst_bits = 0; /* Store count on all updates */
667 rc = roc_npa_pool_create(&sq->aura_handle, blk_sz, NIX_MAX_SQB, &aura,
668 &pool);
669 if (rc)
670 goto fail;
671
672 sq->sqe_mem = plt_zmalloc(blk_sz * NIX_MAX_SQB, blk_sz);
673 if (sq->sqe_mem == NULL) {
674 rc = NIX_ERR_NO_MEM;
675 goto nomem;
676 }
677
678 /* Fill the initial buffers */
679 iova = (uint64_t)sq->sqe_mem;
680 for (count = 0; count < NIX_MAX_SQB; count++) {
681 roc_npa_aura_op_free(sq->aura_handle, 0, iova);
682 iova += blk_sz;
683 }
684
685 if (roc_npa_aura_op_available_wait(sq->aura_handle, NIX_MAX_SQB, 0) !=
686 NIX_MAX_SQB) {
687 plt_err("Failed to free all pointers to the pool");
688 rc = NIX_ERR_NO_MEM;
689 goto npa_fail;
690 }
691
692 roc_npa_aura_op_range_set(sq->aura_handle, (uint64_t)sq->sqe_mem, iova);
693 roc_npa_aura_limit_modify(sq->aura_handle, sq->nb_sqb_bufs);
694 sq->aura_sqb_bufs = NIX_MAX_SQB;
695
696 return rc;
697 npa_fail:
698 plt_free(sq->sqe_mem);
699 nomem:
700 roc_npa_pool_destroy(sq->aura_handle);
701 fail:
702 return rc;
703 }
704
705 static int
sq_cn9k_init(struct nix * nix,struct roc_nix_sq * sq,uint32_t rr_quantum,uint16_t smq)706 sq_cn9k_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum,
707 uint16_t smq)
708 {
709 struct roc_nix *roc_nix = nix_priv_to_roc_nix(nix);
710 struct mbox *mbox = (&nix->dev)->mbox;
711 struct nix_aq_enq_req *aq;
712
713 aq = mbox_alloc_msg_nix_aq_enq(mbox);
714 if (!aq)
715 return -ENOSPC;
716
717 aq->qidx = sq->qid;
718 aq->ctype = NIX_AQ_CTYPE_SQ;
719 aq->op = NIX_AQ_INSTOP_INIT;
720 aq->sq.max_sqe_size = sq->max_sqe_sz;
721
722 aq->sq.max_sqe_size = sq->max_sqe_sz;
723 aq->sq.smq = smq;
724 aq->sq.smq_rr_quantum = rr_quantum;
725 if (roc_nix_is_sdp(roc_nix))
726 aq->sq.default_chan =
727 nix->tx_chan_base + (sq->qid % nix->tx_chan_cnt);
728 else
729 aq->sq.default_chan = nix->tx_chan_base;
730 aq->sq.sqe_stype = NIX_STYPE_STF;
731 aq->sq.ena = 1;
732 aq->sq.sso_ena = !!sq->sso_ena;
733 aq->sq.cq_ena = !!sq->cq_ena;
734 aq->sq.cq = sq->cqid;
735 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8)
736 aq->sq.sqe_stype = NIX_STYPE_STP;
737 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle);
738 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR);
739 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL);
740 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR);
741 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR);
742
743 /* Many to one reduction */
744 /* Assigning QINT 0 to all the SQs, an errata exists where NIXTX can
745 * send incorrect QINT_IDX when reporting queue interrupt (QINT). This
746 * might result in software missing the interrupt.
747 */
748 aq->sq.qint_idx = 0;
749 return 0;
750 }
751
752 static int
sq_cn9k_fini(struct nix * nix,struct roc_nix_sq * sq)753 sq_cn9k_fini(struct nix *nix, struct roc_nix_sq *sq)
754 {
755 struct mbox *mbox = (&nix->dev)->mbox;
756 struct nix_aq_enq_rsp *rsp;
757 struct nix_aq_enq_req *aq;
758 uint16_t sqes_per_sqb;
759 void *sqb_buf;
760 int rc, count;
761
762 aq = mbox_alloc_msg_nix_aq_enq(mbox);
763 if (!aq)
764 return -ENOSPC;
765
766 aq->qidx = sq->qid;
767 aq->ctype = NIX_AQ_CTYPE_SQ;
768 aq->op = NIX_AQ_INSTOP_READ;
769 rc = mbox_process_msg(mbox, (void *)&rsp);
770 if (rc)
771 return rc;
772
773 /* Check if sq is already cleaned up */
774 if (!rsp->sq.ena)
775 return 0;
776
777 /* Disable sq */
778 aq = mbox_alloc_msg_nix_aq_enq(mbox);
779 if (!aq)
780 return -ENOSPC;
781
782 aq->qidx = sq->qid;
783 aq->ctype = NIX_AQ_CTYPE_SQ;
784 aq->op = NIX_AQ_INSTOP_WRITE;
785 aq->sq_mask.ena = ~aq->sq_mask.ena;
786 aq->sq.ena = 0;
787 rc = mbox_process(mbox);
788 if (rc)
789 return rc;
790
791 /* Read SQ and free sqb's */
792 aq = mbox_alloc_msg_nix_aq_enq(mbox);
793 if (!aq)
794 return -ENOSPC;
795
796 aq->qidx = sq->qid;
797 aq->ctype = NIX_AQ_CTYPE_SQ;
798 aq->op = NIX_AQ_INSTOP_READ;
799 rc = mbox_process_msg(mbox, (void *)&rsp);
800 if (rc)
801 return rc;
802
803 if (aq->sq.smq_pend)
804 plt_err("SQ has pending SQE's");
805
806 count = aq->sq.sqb_count;
807 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2;
808 /* Free SQB's that are used */
809 sqb_buf = (void *)rsp->sq.head_sqb;
810 while (count) {
811 void *next_sqb;
812
813 next_sqb = *(void **)((uintptr_t)sqb_buf +
814 (uint32_t)((sqes_per_sqb - 1) *
815 sq->max_sqe_sz));
816 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf);
817 sqb_buf = next_sqb;
818 count--;
819 }
820
821 /* Free next to use sqb */
822 if (rsp->sq.next_sqb)
823 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb);
824 return 0;
825 }
826
827 static int
sq_init(struct nix * nix,struct roc_nix_sq * sq,uint32_t rr_quantum,uint16_t smq)828 sq_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum,
829 uint16_t smq)
830 {
831 struct mbox *mbox = (&nix->dev)->mbox;
832 struct nix_cn10k_aq_enq_req *aq;
833
834 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox);
835 if (!aq)
836 return -ENOSPC;
837
838 aq->qidx = sq->qid;
839 aq->ctype = NIX_AQ_CTYPE_SQ;
840 aq->op = NIX_AQ_INSTOP_INIT;
841 aq->sq.max_sqe_size = sq->max_sqe_sz;
842
843 aq->sq.max_sqe_size = sq->max_sqe_sz;
844 aq->sq.smq = smq;
845 aq->sq.smq_rr_weight = rr_quantum;
846 aq->sq.default_chan = nix->tx_chan_base;
847 aq->sq.sqe_stype = NIX_STYPE_STF;
848 aq->sq.ena = 1;
849 aq->sq.sso_ena = !!sq->sso_ena;
850 aq->sq.cq_ena = !!sq->cq_ena;
851 aq->sq.cq = sq->cqid;
852 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8)
853 aq->sq.sqe_stype = NIX_STYPE_STP;
854 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle);
855 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR);
856 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL);
857 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR);
858 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR);
859
860 /* Assigning QINT 0 to all the SQs, an errata exists where NIXTX can
861 * send incorrect QINT_IDX when reporting queue interrupt (QINT). This
862 * might result in software missing the interrupt.
863 */
864 aq->sq.qint_idx = 0;
865 return 0;
866 }
867
868 static int
sq_fini(struct nix * nix,struct roc_nix_sq * sq)869 sq_fini(struct nix *nix, struct roc_nix_sq *sq)
870 {
871 struct mbox *mbox = (&nix->dev)->mbox;
872 struct nix_cn10k_aq_enq_rsp *rsp;
873 struct nix_cn10k_aq_enq_req *aq;
874 uint16_t sqes_per_sqb;
875 void *sqb_buf;
876 int rc, count;
877
878 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox);
879 if (!aq)
880 return -ENOSPC;
881
882 aq->qidx = sq->qid;
883 aq->ctype = NIX_AQ_CTYPE_SQ;
884 aq->op = NIX_AQ_INSTOP_READ;
885 rc = mbox_process_msg(mbox, (void *)&rsp);
886 if (rc)
887 return rc;
888
889 /* Check if sq is already cleaned up */
890 if (!rsp->sq.ena)
891 return 0;
892
893 /* Disable sq */
894 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox);
895 if (!aq)
896 return -ENOSPC;
897
898 aq->qidx = sq->qid;
899 aq->ctype = NIX_AQ_CTYPE_SQ;
900 aq->op = NIX_AQ_INSTOP_WRITE;
901 aq->sq_mask.ena = ~aq->sq_mask.ena;
902 aq->sq.ena = 0;
903 rc = mbox_process(mbox);
904 if (rc)
905 return rc;
906
907 /* Read SQ and free sqb's */
908 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox);
909 if (!aq)
910 return -ENOSPC;
911
912 aq->qidx = sq->qid;
913 aq->ctype = NIX_AQ_CTYPE_SQ;
914 aq->op = NIX_AQ_INSTOP_READ;
915 rc = mbox_process_msg(mbox, (void *)&rsp);
916 if (rc)
917 return rc;
918
919 if (aq->sq.smq_pend)
920 plt_err("SQ has pending SQE's");
921
922 count = aq->sq.sqb_count;
923 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2;
924 /* Free SQB's that are used */
925 sqb_buf = (void *)rsp->sq.head_sqb;
926 while (count) {
927 void *next_sqb;
928
929 next_sqb = *(void **)((uintptr_t)sqb_buf +
930 (uint32_t)((sqes_per_sqb - 1) *
931 sq->max_sqe_sz));
932 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf);
933 sqb_buf = next_sqb;
934 count--;
935 }
936
937 /* Free next to use sqb */
938 if (rsp->sq.next_sqb)
939 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb);
940 return 0;
941 }
942
943 int
roc_nix_sq_init(struct roc_nix * roc_nix,struct roc_nix_sq * sq)944 roc_nix_sq_init(struct roc_nix *roc_nix, struct roc_nix_sq *sq)
945 {
946 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
947 struct mbox *mbox = (&nix->dev)->mbox;
948 uint16_t qid, smq = UINT16_MAX;
949 uint32_t rr_quantum = 0;
950 int rc;
951
952 if (sq == NULL)
953 return NIX_ERR_PARAM;
954
955 qid = sq->qid;
956 if (qid >= nix->nb_tx_queues)
957 return NIX_ERR_QUEUE_INVALID_RANGE;
958
959 sq->roc_nix = roc_nix;
960 /*
961 * Allocate memory for flow control updates from HW.
962 * Alloc one cache line, so that fits all FC_STYPE modes.
963 */
964 sq->fc = plt_zmalloc(ROC_ALIGN, ROC_ALIGN);
965 if (sq->fc == NULL) {
966 rc = NIX_ERR_NO_MEM;
967 goto fail;
968 }
969
970 rc = sqb_pool_populate(roc_nix, sq);
971 if (rc)
972 goto nomem;
973
974 rc = nix_tm_leaf_data_get(nix, sq->qid, &rr_quantum, &smq);
975 if (rc) {
976 rc = NIX_ERR_TM_LEAF_NODE_GET;
977 goto nomem;
978 }
979
980 /* Init SQ context */
981 if (roc_model_is_cn9k())
982 rc = sq_cn9k_init(nix, sq, rr_quantum, smq);
983 else
984 rc = sq_init(nix, sq, rr_quantum, smq);
985
986 if (rc)
987 goto nomem;
988
989 rc = mbox_process(mbox);
990 if (rc)
991 goto nomem;
992
993 nix->sqs[qid] = sq;
994 sq->io_addr = nix->base + NIX_LF_OP_SENDX(0);
995 /* Evenly distribute LMT slot for each sq */
996 if (roc_model_is_cn9k()) {
997 /* Multiple cores/SQ's can use same LMTLINE safely in CN9K */
998 sq->lmt_addr = (void *)(nix->lmt_base +
999 ((qid & RVU_CN9K_LMT_SLOT_MASK) << 12));
1000 }
1001
1002 rc = nix_tel_node_add_sq(sq);
1003 return rc;
1004 nomem:
1005 plt_free(sq->fc);
1006 fail:
1007 return rc;
1008 }
1009
1010 int
roc_nix_sq_fini(struct roc_nix_sq * sq)1011 roc_nix_sq_fini(struct roc_nix_sq *sq)
1012 {
1013 struct nix *nix;
1014 struct mbox *mbox;
1015 struct ndc_sync_op *ndc_req;
1016 uint16_t qid;
1017 int rc = 0;
1018
1019 if (sq == NULL)
1020 return NIX_ERR_PARAM;
1021
1022 nix = roc_nix_to_nix_priv(sq->roc_nix);
1023 mbox = (&nix->dev)->mbox;
1024
1025 qid = sq->qid;
1026
1027 rc = nix_tm_sq_flush_pre(sq);
1028
1029 /* Release SQ context */
1030 if (roc_model_is_cn9k())
1031 rc |= sq_cn9k_fini(roc_nix_to_nix_priv(sq->roc_nix), sq);
1032 else
1033 rc |= sq_fini(roc_nix_to_nix_priv(sq->roc_nix), sq);
1034
1035 /* Sync NDC-NIX-TX for LF */
1036 ndc_req = mbox_alloc_msg_ndc_sync_op(mbox);
1037 if (ndc_req == NULL)
1038 return -ENOSPC;
1039 ndc_req->nix_lf_tx_sync = 1;
1040 if (mbox_process(mbox))
1041 rc |= NIX_ERR_NDC_SYNC;
1042
1043 rc |= nix_tm_sq_flush_post(sq);
1044
1045 /* Restore limit to max SQB count that the pool was created
1046 * for aura drain to succeed.
1047 */
1048 roc_npa_aura_limit_modify(sq->aura_handle, NIX_MAX_SQB);
1049 rc |= roc_npa_pool_destroy(sq->aura_handle);
1050 plt_free(sq->fc);
1051 plt_free(sq->sqe_mem);
1052 nix->sqs[qid] = NULL;
1053
1054 return rc;
1055 }
1056
1057 void
roc_nix_cq_head_tail_get(struct roc_nix * roc_nix,uint16_t qid,uint32_t * head,uint32_t * tail)1058 roc_nix_cq_head_tail_get(struct roc_nix *roc_nix, uint16_t qid, uint32_t *head,
1059 uint32_t *tail)
1060 {
1061 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1062 uint64_t reg, val;
1063 int64_t *addr;
1064
1065 if (head == NULL || tail == NULL)
1066 return;
1067
1068 reg = (((uint64_t)qid) << 32);
1069 addr = (int64_t *)(nix->base + NIX_LF_CQ_OP_STATUS);
1070 val = roc_atomic64_add_nosync(reg, addr);
1071 if (val &
1072 (BIT_ULL(NIX_CQ_OP_STAT_OP_ERR) | BIT_ULL(NIX_CQ_OP_STAT_CQ_ERR)))
1073 val = 0;
1074
1075 *tail = (uint32_t)(val & 0xFFFFF);
1076 *head = (uint32_t)((val >> 20) & 0xFFFFF);
1077 }
1078
1079 void
roc_nix_sq_head_tail_get(struct roc_nix * roc_nix,uint16_t qid,uint32_t * head,uint32_t * tail)1080 roc_nix_sq_head_tail_get(struct roc_nix *roc_nix, uint16_t qid, uint32_t *head,
1081 uint32_t *tail)
1082 {
1083 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1084 struct roc_nix_sq *sq = nix->sqs[qid];
1085 uint16_t sqes_per_sqb, sqb_cnt;
1086 uint64_t reg, val;
1087 int64_t *addr;
1088
1089 if (head == NULL || tail == NULL)
1090 return;
1091
1092 reg = (((uint64_t)qid) << 32);
1093 addr = (int64_t *)(nix->base + NIX_LF_SQ_OP_STATUS);
1094 val = roc_atomic64_add_nosync(reg, addr);
1095 if (val & BIT_ULL(NIX_CQ_OP_STAT_OP_ERR)) {
1096 val = 0;
1097 return;
1098 }
1099
1100 *tail = (uint32_t)((val >> 28) & 0x3F);
1101 *head = (uint32_t)((val >> 20) & 0x3F);
1102 sqb_cnt = (uint16_t)(val & 0xFFFF);
1103
1104 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2;
1105
1106 /* Update tail index as per used sqb count */
1107 *tail += (sqes_per_sqb * (sqb_cnt - 1));
1108 }
1109