1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
3 */
4
5 #include <unistd.h>
6
7 #include <rte_eal.h>
8 #include <rte_mempool.h>
9 #include <rte_mbuf.h>
10 #include <rte_io.h>
11 #include <rte_net.h>
12 #include <ethdev_pci.h>
13
14 #include "otx_ep_common.h"
15 #include "otx_ep_vf.h"
16 #include "otx2_ep_vf.h"
17 #include "otx_ep_rxtx.h"
18
19 /* SDP_LENGTH_S specifies packet length and is of 8-byte size */
20 #define INFO_SIZE 8
21 #define DROQ_REFILL_THRESHOLD 16
22
23 static void
otx_ep_dmazone_free(const struct rte_memzone * mz)24 otx_ep_dmazone_free(const struct rte_memzone *mz)
25 {
26 const struct rte_memzone *mz_tmp;
27 int ret = 0;
28
29 if (mz == NULL) {
30 otx_ep_err("Memzone: NULL\n");
31 return;
32 }
33
34 mz_tmp = rte_memzone_lookup(mz->name);
35 if (mz_tmp == NULL) {
36 otx_ep_err("Memzone %s Not Found\n", mz->name);
37 return;
38 }
39
40 ret = rte_memzone_free(mz);
41 if (ret)
42 otx_ep_err("Memzone free failed : ret = %d\n", ret);
43 }
44
45 /* Free IQ resources */
46 int
otx_ep_delete_iqs(struct otx_ep_device * otx_ep,uint32_t iq_no)47 otx_ep_delete_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no)
48 {
49 struct otx_ep_instr_queue *iq;
50
51 iq = otx_ep->instr_queue[iq_no];
52 if (iq == NULL) {
53 otx_ep_err("Invalid IQ[%d]\n", iq_no);
54 return -EINVAL;
55 }
56
57 rte_free(iq->req_list);
58 iq->req_list = NULL;
59
60 if (iq->iq_mz) {
61 otx_ep_dmazone_free(iq->iq_mz);
62 iq->iq_mz = NULL;
63 }
64
65 rte_free(otx_ep->instr_queue[iq_no]);
66 otx_ep->instr_queue[iq_no] = NULL;
67
68 otx_ep->nb_tx_queues--;
69
70 otx_ep_info("IQ[%d] is deleted\n", iq_no);
71
72 return 0;
73 }
74
75 /* IQ initialization */
76 static int
otx_ep_init_instr_queue(struct otx_ep_device * otx_ep,int iq_no,int num_descs,unsigned int socket_id)77 otx_ep_init_instr_queue(struct otx_ep_device *otx_ep, int iq_no, int num_descs,
78 unsigned int socket_id)
79 {
80 const struct otx_ep_config *conf;
81 struct otx_ep_instr_queue *iq;
82 uint32_t q_size;
83
84 conf = otx_ep->conf;
85 iq = otx_ep->instr_queue[iq_no];
86 q_size = conf->iq.instr_type * num_descs;
87
88 /* IQ memory creation for Instruction submission to OCTEON 9 */
89 iq->iq_mz = rte_eth_dma_zone_reserve(otx_ep->eth_dev,
90 "instr_queue", iq_no, q_size,
91 OTX_EP_PCI_RING_ALIGN,
92 socket_id);
93 if (iq->iq_mz == NULL) {
94 otx_ep_err("IQ[%d] memzone alloc failed\n", iq_no);
95 goto iq_init_fail;
96 }
97
98 iq->base_addr_dma = iq->iq_mz->iova;
99 iq->base_addr = (uint8_t *)iq->iq_mz->addr;
100
101 if (num_descs & (num_descs - 1)) {
102 otx_ep_err("IQ[%d] descs not in power of 2\n", iq_no);
103 goto iq_init_fail;
104 }
105
106 iq->nb_desc = num_descs;
107
108 /* Create a IQ request list to hold requests that have been
109 * posted to OCTEON 9. This list will be used for freeing the IQ
110 * data buffer(s) later once the OCTEON 9 fetched the requests.
111 */
112 iq->req_list = rte_zmalloc_socket("request_list",
113 (iq->nb_desc * OTX_EP_IQREQ_LIST_SIZE),
114 RTE_CACHE_LINE_SIZE,
115 rte_socket_id());
116 if (iq->req_list == NULL) {
117 otx_ep_err("IQ[%d] req_list alloc failed\n", iq_no);
118 goto iq_init_fail;
119 }
120
121 otx_ep_info("IQ[%d]: base: %p basedma: %lx count: %d\n",
122 iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma,
123 iq->nb_desc);
124
125 iq->otx_ep_dev = otx_ep;
126 iq->q_no = iq_no;
127 iq->fill_cnt = 0;
128 iq->host_write_index = 0;
129 iq->otx_read_index = 0;
130 iq->flush_index = 0;
131 iq->instr_pending = 0;
132
133 otx_ep->io_qmask.iq |= (1ull << iq_no);
134
135 /* Set 32B/64B mode for each input queue */
136 if (conf->iq.instr_type == 64)
137 otx_ep->io_qmask.iq64B |= (1ull << iq_no);
138
139 iq->iqcmd_64B = (conf->iq.instr_type == 64);
140
141 /* Set up IQ registers */
142 otx_ep->fn_list.setup_iq_regs(otx_ep, iq_no);
143
144 return 0;
145
146 iq_init_fail:
147 return -ENOMEM;
148 }
149
150 int
otx_ep_setup_iqs(struct otx_ep_device * otx_ep,uint32_t iq_no,int num_descs,unsigned int socket_id)151 otx_ep_setup_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no, int num_descs,
152 unsigned int socket_id)
153 {
154 struct otx_ep_instr_queue *iq;
155
156 iq = (struct otx_ep_instr_queue *)rte_zmalloc("otx_ep_IQ", sizeof(*iq),
157 RTE_CACHE_LINE_SIZE);
158 if (iq == NULL)
159 return -ENOMEM;
160
161 otx_ep->instr_queue[iq_no] = iq;
162
163 if (otx_ep_init_instr_queue(otx_ep, iq_no, num_descs, socket_id)) {
164 otx_ep_err("IQ init is failed\n");
165 goto delete_IQ;
166 }
167 otx_ep->nb_tx_queues++;
168
169 otx_ep_info("IQ[%d] is created.\n", iq_no);
170
171 return 0;
172
173 delete_IQ:
174 otx_ep_delete_iqs(otx_ep, iq_no);
175 return -ENOMEM;
176 }
177
178 static void
otx_ep_droq_reset_indices(struct otx_ep_droq * droq)179 otx_ep_droq_reset_indices(struct otx_ep_droq *droq)
180 {
181 droq->read_idx = 0;
182 droq->write_idx = 0;
183 droq->refill_idx = 0;
184 droq->refill_count = 0;
185 droq->last_pkt_count = 0;
186 droq->pkts_pending = 0;
187 }
188
189 static void
otx_ep_droq_destroy_ring_buffers(struct otx_ep_droq * droq)190 otx_ep_droq_destroy_ring_buffers(struct otx_ep_droq *droq)
191 {
192 uint32_t idx;
193
194 for (idx = 0; idx < droq->nb_desc; idx++) {
195 if (droq->recv_buf_list[idx]) {
196 rte_pktmbuf_free(droq->recv_buf_list[idx]);
197 droq->recv_buf_list[idx] = NULL;
198 }
199 }
200
201 otx_ep_droq_reset_indices(droq);
202 }
203
204 /* Free OQs resources */
205 int
otx_ep_delete_oqs(struct otx_ep_device * otx_ep,uint32_t oq_no)206 otx_ep_delete_oqs(struct otx_ep_device *otx_ep, uint32_t oq_no)
207 {
208 struct otx_ep_droq *droq;
209
210 droq = otx_ep->droq[oq_no];
211 if (droq == NULL) {
212 otx_ep_err("Invalid droq[%d]\n", oq_no);
213 return -EINVAL;
214 }
215
216 otx_ep_droq_destroy_ring_buffers(droq);
217 rte_free(droq->recv_buf_list);
218 droq->recv_buf_list = NULL;
219
220 if (droq->desc_ring_mz) {
221 otx_ep_dmazone_free(droq->desc_ring_mz);
222 droq->desc_ring_mz = NULL;
223 }
224
225 memset(droq, 0, OTX_EP_DROQ_SIZE);
226
227 rte_free(otx_ep->droq[oq_no]);
228 otx_ep->droq[oq_no] = NULL;
229
230 otx_ep->nb_rx_queues--;
231
232 otx_ep_info("OQ[%d] is deleted\n", oq_no);
233 return 0;
234 }
235
236 static int
otx_ep_droq_setup_ring_buffers(struct otx_ep_droq * droq)237 otx_ep_droq_setup_ring_buffers(struct otx_ep_droq *droq)
238 {
239 struct otx_ep_droq_desc *desc_ring = droq->desc_ring;
240 struct otx_ep_droq_info *info;
241 struct rte_mbuf *buf;
242 uint32_t idx;
243
244 for (idx = 0; idx < droq->nb_desc; idx++) {
245 buf = rte_pktmbuf_alloc(droq->mpool);
246 if (buf == NULL) {
247 otx_ep_err("OQ buffer alloc failed\n");
248 droq->stats.rx_alloc_failure++;
249 return -ENOMEM;
250 }
251
252 droq->recv_buf_list[idx] = buf;
253 info = rte_pktmbuf_mtod(buf, struct otx_ep_droq_info *);
254 memset(info, 0, sizeof(*info));
255 desc_ring[idx].buffer_ptr = rte_mbuf_data_iova_default(buf);
256 }
257
258 otx_ep_droq_reset_indices(droq);
259
260 return 0;
261 }
262
263 /* OQ initialization */
264 static int
otx_ep_init_droq(struct otx_ep_device * otx_ep,uint32_t q_no,uint32_t num_descs,uint32_t desc_size,struct rte_mempool * mpool,unsigned int socket_id)265 otx_ep_init_droq(struct otx_ep_device *otx_ep, uint32_t q_no,
266 uint32_t num_descs, uint32_t desc_size,
267 struct rte_mempool *mpool, unsigned int socket_id)
268 {
269 const struct otx_ep_config *conf = otx_ep->conf;
270 uint32_t c_refill_threshold;
271 struct otx_ep_droq *droq;
272 uint32_t desc_ring_size;
273
274 otx_ep_info("OQ[%d] Init start\n", q_no);
275
276 droq = otx_ep->droq[q_no];
277 droq->otx_ep_dev = otx_ep;
278 droq->q_no = q_no;
279 droq->mpool = mpool;
280
281 droq->nb_desc = num_descs;
282 droq->buffer_size = desc_size;
283 c_refill_threshold = RTE_MAX(conf->oq.refill_threshold,
284 droq->nb_desc / 2);
285
286 /* OQ desc_ring set up */
287 desc_ring_size = droq->nb_desc * OTX_EP_DROQ_DESC_SIZE;
288 droq->desc_ring_mz = rte_eth_dma_zone_reserve(otx_ep->eth_dev, "droq",
289 q_no, desc_ring_size,
290 OTX_EP_PCI_RING_ALIGN,
291 socket_id);
292
293 if (droq->desc_ring_mz == NULL) {
294 otx_ep_err("OQ:%d desc_ring allocation failed\n", q_no);
295 goto init_droq_fail;
296 }
297
298 droq->desc_ring_dma = droq->desc_ring_mz->iova;
299 droq->desc_ring = (struct otx_ep_droq_desc *)droq->desc_ring_mz->addr;
300
301 otx_ep_dbg("OQ[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
302 q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma);
303 otx_ep_dbg("OQ[%d]: num_desc: %d\n", q_no, droq->nb_desc);
304
305 /* OQ buf_list set up */
306 droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list",
307 (droq->nb_desc * sizeof(struct rte_mbuf *)),
308 RTE_CACHE_LINE_SIZE, socket_id);
309 if (droq->recv_buf_list == NULL) {
310 otx_ep_err("OQ recv_buf_list alloc failed\n");
311 goto init_droq_fail;
312 }
313
314 if (otx_ep_droq_setup_ring_buffers(droq))
315 goto init_droq_fail;
316
317 droq->refill_threshold = c_refill_threshold;
318
319 /* Set up OQ registers */
320 otx_ep->fn_list.setup_oq_regs(otx_ep, q_no);
321
322 otx_ep->io_qmask.oq |= (1ull << q_no);
323
324 return 0;
325
326 init_droq_fail:
327 return -ENOMEM;
328 }
329
330 /* OQ configuration and setup */
331 int
otx_ep_setup_oqs(struct otx_ep_device * otx_ep,int oq_no,int num_descs,int desc_size,struct rte_mempool * mpool,unsigned int socket_id)332 otx_ep_setup_oqs(struct otx_ep_device *otx_ep, int oq_no, int num_descs,
333 int desc_size, struct rte_mempool *mpool,
334 unsigned int socket_id)
335 {
336 struct otx_ep_droq *droq;
337
338 /* Allocate new droq. */
339 droq = (struct otx_ep_droq *)rte_zmalloc("otx_ep_OQ",
340 sizeof(*droq), RTE_CACHE_LINE_SIZE);
341 if (droq == NULL) {
342 otx_ep_err("Droq[%d] Creation Failed\n", oq_no);
343 return -ENOMEM;
344 }
345 otx_ep->droq[oq_no] = droq;
346
347 if (otx_ep_init_droq(otx_ep, oq_no, num_descs, desc_size, mpool,
348 socket_id)) {
349 otx_ep_err("Droq[%d] Initialization failed\n", oq_no);
350 goto delete_OQ;
351 }
352 otx_ep_info("OQ[%d] is created.\n", oq_no);
353
354 otx_ep->nb_rx_queues++;
355
356 return 0;
357
358 delete_OQ:
359 otx_ep_delete_oqs(otx_ep, oq_no);
360 return -ENOMEM;
361 }
362
363 static inline void
otx_ep_iqreq_delete(struct otx_ep_instr_queue * iq,uint32_t idx)364 otx_ep_iqreq_delete(struct otx_ep_instr_queue *iq, uint32_t idx)
365 {
366 uint32_t reqtype;
367 void *buf;
368 struct otx_ep_buf_free_info *finfo;
369
370 buf = iq->req_list[idx].buf;
371 reqtype = iq->req_list[idx].reqtype;
372
373 switch (reqtype) {
374 case OTX_EP_REQTYPE_NORESP_NET:
375 rte_pktmbuf_free((struct rte_mbuf *)buf);
376 otx_ep_dbg("IQ buffer freed at idx[%d]\n", idx);
377 break;
378
379 case OTX_EP_REQTYPE_NORESP_GATHER:
380 finfo = (struct otx_ep_buf_free_info *)buf;
381 /* This will take care of multiple segments also */
382 rte_pktmbuf_free(finfo->mbuf);
383 rte_free(finfo->g.sg);
384 rte_free(finfo);
385 break;
386
387 case OTX_EP_REQTYPE_NONE:
388 default:
389 otx_ep_info("This iqreq mode is not supported:%d\n", reqtype);
390 }
391
392 /* Reset the request list at this index */
393 iq->req_list[idx].buf = NULL;
394 iq->req_list[idx].reqtype = 0;
395 }
396
397 static inline void
otx_ep_iqreq_add(struct otx_ep_instr_queue * iq,void * buf,uint32_t reqtype,int index)398 otx_ep_iqreq_add(struct otx_ep_instr_queue *iq, void *buf,
399 uint32_t reqtype, int index)
400 {
401 iq->req_list[index].buf = buf;
402 iq->req_list[index].reqtype = reqtype;
403 }
404
405 static uint32_t
otx_vf_update_read_index(struct otx_ep_instr_queue * iq)406 otx_vf_update_read_index(struct otx_ep_instr_queue *iq)
407 {
408 uint32_t new_idx = rte_read32(iq->inst_cnt_reg);
409 if (unlikely(new_idx == 0xFFFFFFFFU))
410 rte_write32(new_idx, iq->inst_cnt_reg);
411 /* Modulo of the new index with the IQ size will give us
412 * the new index.
413 */
414 new_idx &= (iq->nb_desc - 1);
415
416 return new_idx;
417 }
418
419 static void
otx_ep_flush_iq(struct otx_ep_instr_queue * iq)420 otx_ep_flush_iq(struct otx_ep_instr_queue *iq)
421 {
422 uint32_t instr_processed = 0;
423
424 iq->otx_read_index = otx_vf_update_read_index(iq);
425 while (iq->flush_index != iq->otx_read_index) {
426 /* Free the IQ data buffer to the pool */
427 otx_ep_iqreq_delete(iq, iq->flush_index);
428 iq->flush_index =
429 otx_ep_incr_index(iq->flush_index, 1, iq->nb_desc);
430
431 instr_processed++;
432 }
433
434 iq->stats.instr_processed = instr_processed;
435 iq->instr_pending -= instr_processed;
436 }
437
438 static inline void
otx_ep_ring_doorbell(struct otx_ep_device * otx_ep __rte_unused,struct otx_ep_instr_queue * iq)439 otx_ep_ring_doorbell(struct otx_ep_device *otx_ep __rte_unused,
440 struct otx_ep_instr_queue *iq)
441 {
442 rte_wmb();
443 rte_write64(iq->fill_cnt, iq->doorbell_reg);
444 iq->fill_cnt = 0;
445 }
446
447 static inline int
post_iqcmd(struct otx_ep_instr_queue * iq,uint8_t * iqcmd)448 post_iqcmd(struct otx_ep_instr_queue *iq, uint8_t *iqcmd)
449 {
450 uint8_t *iqptr, cmdsize;
451
452 /* This ensures that the read index does not wrap around to
453 * the same position if queue gets full before OCTEON 9 could
454 * fetch any instr.
455 */
456 if (iq->instr_pending > (iq->nb_desc - 1))
457 return OTX_EP_IQ_SEND_FAILED;
458
459 /* Copy cmd into iq */
460 cmdsize = 64;
461 iqptr = iq->base_addr + (iq->host_write_index << 6);
462
463 rte_memcpy(iqptr, iqcmd, cmdsize);
464
465 /* Increment the host write index */
466 iq->host_write_index =
467 otx_ep_incr_index(iq->host_write_index, 1, iq->nb_desc);
468
469 iq->fill_cnt++;
470
471 /* Flush the command into memory. We need to be sure the data
472 * is in memory before indicating that the instruction is
473 * pending.
474 */
475 iq->instr_pending++;
476 /* OTX_EP_IQ_SEND_SUCCESS */
477 return 0;
478 }
479
480
481 static int
otx_ep_send_data(struct otx_ep_device * otx_ep,struct otx_ep_instr_queue * iq,void * cmd,int dbell)482 otx_ep_send_data(struct otx_ep_device *otx_ep, struct otx_ep_instr_queue *iq,
483 void *cmd, int dbell)
484 {
485 uint32_t ret;
486
487 /* Submit IQ command */
488 ret = post_iqcmd(iq, cmd);
489
490 if (ret == OTX_EP_IQ_SEND_SUCCESS) {
491 if (dbell)
492 otx_ep_ring_doorbell(otx_ep, iq);
493 iq->stats.instr_posted++;
494
495 } else {
496 iq->stats.instr_dropped++;
497 if (iq->fill_cnt)
498 otx_ep_ring_doorbell(otx_ep, iq);
499 }
500 return ret;
501 }
502
503 static inline void
set_sg_size(struct otx_ep_sg_entry * sg_entry,uint16_t size,uint32_t pos)504 set_sg_size(struct otx_ep_sg_entry *sg_entry, uint16_t size, uint32_t pos)
505 {
506 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
507 sg_entry->u.size[pos] = size;
508 #elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
509 sg_entry->u.size[3 - pos] = size;
510 #endif
511 }
512
513 /* Enqueue requests/packets to OTX_EP IQ queue.
514 * returns number of requests enqueued successfully
515 */
516 uint16_t
otx_ep_xmit_pkts(void * tx_queue,struct rte_mbuf ** pkts,uint16_t nb_pkts)517 otx_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
518 {
519 struct otx_ep_instr_64B iqcmd;
520 struct otx_ep_instr_queue *iq;
521 struct otx_ep_device *otx_ep;
522 struct rte_mbuf *m;
523
524 uint32_t iqreq_type, sgbuf_sz;
525 int dbell, index, count = 0;
526 unsigned int pkt_len, i;
527 int gather, gsz;
528 void *iqreq_buf;
529 uint64_t dptr;
530
531 iq = (struct otx_ep_instr_queue *)tx_queue;
532 otx_ep = iq->otx_ep_dev;
533
534 iqcmd.ih.u64 = 0;
535 iqcmd.pki_ih3.u64 = 0;
536 iqcmd.irh.u64 = 0;
537
538 /* ih invars */
539 iqcmd.ih.s.fsz = OTX_EP_FSZ;
540 iqcmd.ih.s.pkind = otx_ep->pkind; /* The SDK decided PKIND value */
541
542 /* pki ih3 invars */
543 iqcmd.pki_ih3.s.w = 1;
544 iqcmd.pki_ih3.s.utt = 1;
545 iqcmd.pki_ih3.s.tagtype = ORDERED_TAG;
546 /* sl will be sizeof(pki_ih3) */
547 iqcmd.pki_ih3.s.sl = OTX_EP_FSZ + OTX_CUST_DATA_LEN;
548
549 /* irh invars */
550 iqcmd.irh.s.opcode = OTX_EP_NW_PKT_OP;
551
552 for (i = 0; i < nb_pkts; i++) {
553 m = pkts[i];
554 if (m->nb_segs == 1) {
555 /* dptr */
556 dptr = rte_mbuf_data_iova(m);
557 pkt_len = rte_pktmbuf_data_len(m);
558 iqreq_buf = m;
559 iqreq_type = OTX_EP_REQTYPE_NORESP_NET;
560 gather = 0;
561 gsz = 0;
562 } else {
563 struct otx_ep_buf_free_info *finfo;
564 int j, frags, num_sg;
565
566 if (!(otx_ep->tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS))
567 goto xmit_fail;
568
569 finfo = (struct otx_ep_buf_free_info *)rte_malloc(NULL,
570 sizeof(*finfo), 0);
571 if (finfo == NULL) {
572 otx_ep_err("free buffer alloc failed\n");
573 goto xmit_fail;
574 }
575 num_sg = (m->nb_segs + 3) / 4;
576 sgbuf_sz = sizeof(struct otx_ep_sg_entry) * num_sg;
577 finfo->g.sg =
578 rte_zmalloc(NULL, sgbuf_sz, OTX_EP_SG_ALIGN);
579 if (finfo->g.sg == NULL) {
580 rte_free(finfo);
581 otx_ep_err("sg entry alloc failed\n");
582 goto xmit_fail;
583 }
584 gather = 1;
585 gsz = m->nb_segs;
586 finfo->g.num_sg = num_sg;
587 finfo->g.sg[0].ptr[0] = rte_mbuf_data_iova(m);
588 set_sg_size(&finfo->g.sg[0], m->data_len, 0);
589 pkt_len = m->data_len;
590 finfo->mbuf = m;
591
592 frags = m->nb_segs - 1;
593 j = 1;
594 m = m->next;
595 while (frags--) {
596 finfo->g.sg[(j >> 2)].ptr[(j & 3)] =
597 rte_mbuf_data_iova(m);
598 set_sg_size(&finfo->g.sg[(j >> 2)],
599 m->data_len, (j & 3));
600 pkt_len += m->data_len;
601 j++;
602 m = m->next;
603 }
604 dptr = rte_mem_virt2iova(finfo->g.sg);
605 iqreq_buf = finfo;
606 iqreq_type = OTX_EP_REQTYPE_NORESP_GATHER;
607 if (pkt_len > OTX_EP_MAX_PKT_SZ) {
608 rte_free(finfo->g.sg);
609 rte_free(finfo);
610 otx_ep_err("failed\n");
611 goto xmit_fail;
612 }
613 }
614 /* ih vars */
615 iqcmd.ih.s.tlen = pkt_len + iqcmd.ih.s.fsz;
616 iqcmd.ih.s.gather = gather;
617 iqcmd.ih.s.gsz = gsz;
618
619 iqcmd.dptr = dptr;
620 otx_ep_swap_8B_data(&iqcmd.irh.u64, 1);
621
622 #ifdef OTX_EP_IO_DEBUG
623 otx_ep_dbg("After swapping\n");
624 otx_ep_dbg("Word0 [dptr]: 0x%016lx\n",
625 (unsigned long)iqcmd.dptr);
626 otx_ep_dbg("Word1 [ihtx]: 0x%016lx\n", (unsigned long)iqcmd.ih);
627 otx_ep_dbg("Word2 [pki_ih3]: 0x%016lx\n",
628 (unsigned long)iqcmd.pki_ih3);
629 otx_ep_dbg("Word3 [rptr]: 0x%016lx\n",
630 (unsigned long)iqcmd.rptr);
631 otx_ep_dbg("Word4 [irh]: 0x%016lx\n", (unsigned long)iqcmd.irh);
632 otx_ep_dbg("Word5 [exhdr[0]]: 0x%016lx\n",
633 (unsigned long)iqcmd.exhdr[0]);
634 rte_pktmbuf_dump(stdout, m, rte_pktmbuf_pkt_len(m));
635 #endif
636 dbell = (i == (unsigned int)(nb_pkts - 1)) ? 1 : 0;
637 index = iq->host_write_index;
638 if (otx_ep_send_data(otx_ep, iq, &iqcmd, dbell))
639 goto xmit_fail;
640 otx_ep_iqreq_add(iq, iqreq_buf, iqreq_type, index);
641 iq->stats.tx_pkts++;
642 iq->stats.tx_bytes += pkt_len;
643 count++;
644 }
645
646 xmit_fail:
647 if (iq->instr_pending >= OTX_EP_MAX_INSTR)
648 otx_ep_flush_iq(iq);
649
650 /* Return no# of instructions posted successfully. */
651 return count;
652 }
653
654 /* Enqueue requests/packets to OTX_EP IQ queue.
655 * returns number of requests enqueued successfully
656 */
657 uint16_t
otx2_ep_xmit_pkts(void * tx_queue,struct rte_mbuf ** pkts,uint16_t nb_pkts)658 otx2_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
659 {
660 struct otx2_ep_instr_64B iqcmd2;
661 struct otx_ep_instr_queue *iq;
662 struct otx_ep_device *otx_ep;
663 uint64_t dptr;
664 int count = 0;
665 unsigned int i;
666 struct rte_mbuf *m;
667 unsigned int pkt_len;
668 void *iqreq_buf;
669 uint32_t iqreq_type, sgbuf_sz;
670 int gather, gsz;
671 int dbell;
672 int index;
673
674 iq = (struct otx_ep_instr_queue *)tx_queue;
675 otx_ep = iq->otx_ep_dev;
676
677 iqcmd2.ih.u64 = 0;
678 iqcmd2.irh.u64 = 0;
679
680 /* ih invars */
681 iqcmd2.ih.s.fsz = OTX2_EP_FSZ;
682 iqcmd2.ih.s.pkind = otx_ep->pkind; /* The SDK decided PKIND value */
683 /* irh invars */
684 iqcmd2.irh.s.opcode = OTX_EP_NW_PKT_OP;
685
686 for (i = 0; i < nb_pkts; i++) {
687 m = pkts[i];
688 if (m->nb_segs == 1) {
689 /* dptr */
690 dptr = rte_mbuf_data_iova(m);
691 pkt_len = rte_pktmbuf_data_len(m);
692 iqreq_buf = m;
693 iqreq_type = OTX_EP_REQTYPE_NORESP_NET;
694 gather = 0;
695 gsz = 0;
696 } else {
697 struct otx_ep_buf_free_info *finfo;
698 int j, frags, num_sg;
699
700 if (!(otx_ep->tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS))
701 goto xmit_fail;
702
703 finfo = (struct otx_ep_buf_free_info *)
704 rte_malloc(NULL, sizeof(*finfo), 0);
705 if (finfo == NULL) {
706 otx_ep_err("free buffer alloc failed\n");
707 goto xmit_fail;
708 }
709 num_sg = (m->nb_segs + 3) / 4;
710 sgbuf_sz = sizeof(struct otx_ep_sg_entry) * num_sg;
711 finfo->g.sg =
712 rte_zmalloc(NULL, sgbuf_sz, OTX_EP_SG_ALIGN);
713 if (finfo->g.sg == NULL) {
714 rte_free(finfo);
715 otx_ep_err("sg entry alloc failed\n");
716 goto xmit_fail;
717 }
718 gather = 1;
719 gsz = m->nb_segs;
720 finfo->g.num_sg = num_sg;
721 finfo->g.sg[0].ptr[0] = rte_mbuf_data_iova(m);
722 set_sg_size(&finfo->g.sg[0], m->data_len, 0);
723 pkt_len = m->data_len;
724 finfo->mbuf = m;
725
726 frags = m->nb_segs - 1;
727 j = 1;
728 m = m->next;
729 while (frags--) {
730 finfo->g.sg[(j >> 2)].ptr[(j & 3)] =
731 rte_mbuf_data_iova(m);
732 set_sg_size(&finfo->g.sg[(j >> 2)],
733 m->data_len, (j & 3));
734 pkt_len += m->data_len;
735 j++;
736 m = m->next;
737 }
738 dptr = rte_mem_virt2iova(finfo->g.sg);
739 iqreq_buf = finfo;
740 iqreq_type = OTX_EP_REQTYPE_NORESP_GATHER;
741 if (pkt_len > OTX_EP_MAX_PKT_SZ) {
742 rte_free(finfo->g.sg);
743 rte_free(finfo);
744 otx_ep_err("failed\n");
745 goto xmit_fail;
746 }
747 }
748 /* ih vars */
749 iqcmd2.ih.s.tlen = pkt_len + iqcmd2.ih.s.fsz;
750 iqcmd2.ih.s.gather = gather;
751 iqcmd2.ih.s.gsz = gsz;
752 iqcmd2.dptr = dptr;
753 otx_ep_swap_8B_data(&iqcmd2.irh.u64, 1);
754
755 #ifdef OTX_EP_IO_DEBUG
756 otx_ep_dbg("After swapping\n");
757 otx_ep_dbg("Word0 [dptr]: 0x%016lx\n",
758 (unsigned long)iqcmd.dptr);
759 otx_ep_dbg("Word1 [ihtx]: 0x%016lx\n", (unsigned long)iqcmd.ih);
760 otx_ep_dbg("Word2 [pki_ih3]: 0x%016lx\n",
761 (unsigned long)iqcmd.pki_ih3);
762 otx_ep_dbg("Word3 [rptr]: 0x%016lx\n",
763 (unsigned long)iqcmd.rptr);
764 otx_ep_dbg("Word4 [irh]: 0x%016lx\n", (unsigned long)iqcmd.irh);
765 otx_ep_dbg("Word5 [exhdr[0]]: 0x%016lx\n",
766 (unsigned long)iqcmd.exhdr[0]);
767 #endif
768 index = iq->host_write_index;
769 dbell = (i == (unsigned int)(nb_pkts - 1)) ? 1 : 0;
770 if (otx_ep_send_data(otx_ep, iq, &iqcmd2, dbell))
771 goto xmit_fail;
772 otx_ep_iqreq_add(iq, iqreq_buf, iqreq_type, index);
773 iq->stats.tx_pkts++;
774 iq->stats.tx_bytes += pkt_len;
775 count++;
776 }
777
778 xmit_fail:
779 if (iq->instr_pending >= OTX_EP_MAX_INSTR)
780 otx_ep_flush_iq(iq);
781
782 /* Return no# of instructions posted successfully. */
783 return count;
784 }
785
786 static uint32_t
otx_ep_droq_refill(struct otx_ep_droq * droq)787 otx_ep_droq_refill(struct otx_ep_droq *droq)
788 {
789 struct otx_ep_droq_desc *desc_ring;
790 struct otx_ep_droq_info *info;
791 struct rte_mbuf *buf = NULL;
792 uint32_t desc_refilled = 0;
793
794 desc_ring = droq->desc_ring;
795
796 while (droq->refill_count && (desc_refilled < droq->nb_desc)) {
797 /* If a valid buffer exists (happens if there is no dispatch),
798 * reuse the buffer, else allocate.
799 */
800 if (droq->recv_buf_list[droq->refill_idx] != NULL)
801 break;
802
803 buf = rte_pktmbuf_alloc(droq->mpool);
804 /* If a buffer could not be allocated, no point in
805 * continuing
806 */
807 if (buf == NULL) {
808 droq->stats.rx_alloc_failure++;
809 break;
810 }
811 info = rte_pktmbuf_mtod(buf, struct otx_ep_droq_info *);
812 memset(info, 0, sizeof(*info));
813
814 droq->recv_buf_list[droq->refill_idx] = buf;
815 desc_ring[droq->refill_idx].buffer_ptr =
816 rte_mbuf_data_iova_default(buf);
817
818
819 droq->refill_idx = otx_ep_incr_index(droq->refill_idx, 1,
820 droq->nb_desc);
821
822 desc_refilled++;
823 droq->refill_count--;
824 }
825
826 return desc_refilled;
827 }
828
829 static struct rte_mbuf *
otx_ep_droq_read_packet(struct otx_ep_device * otx_ep,struct otx_ep_droq * droq,int next_fetch)830 otx_ep_droq_read_packet(struct otx_ep_device *otx_ep,
831 struct otx_ep_droq *droq, int next_fetch)
832 {
833 volatile struct otx_ep_droq_info *info;
834 struct rte_mbuf *droq_pkt2 = NULL;
835 struct rte_mbuf *droq_pkt = NULL;
836 struct rte_net_hdr_lens hdr_lens;
837 struct otx_ep_droq_info *info2;
838 uint64_t total_pkt_len;
839 uint32_t pkt_len = 0;
840 int next_idx;
841
842 droq_pkt = droq->recv_buf_list[droq->read_idx];
843 droq_pkt2 = droq->recv_buf_list[droq->read_idx];
844 info = rte_pktmbuf_mtod(droq_pkt, struct otx_ep_droq_info *);
845 /* make sure info is available */
846 rte_rmb();
847 if (unlikely(!info->length)) {
848 int retry = OTX_EP_MAX_DELAYED_PKT_RETRIES;
849 /* otx_ep_dbg("OCTEON DROQ[%d]: read_idx: %d; Data not ready "
850 * "yet, Retry; pending=%lu\n", droq->q_no, droq->read_idx,
851 * droq->pkts_pending);
852 */
853 droq->stats.pkts_delayed_data++;
854 while (retry && !info->length)
855 retry--;
856 if (!retry && !info->length) {
857 otx_ep_err("OCTEON DROQ[%d]: read_idx: %d; Retry failed !!\n",
858 droq->q_no, droq->read_idx);
859 /* May be zero length packet; drop it */
860 rte_pktmbuf_free(droq_pkt);
861 droq->recv_buf_list[droq->read_idx] = NULL;
862 droq->read_idx = otx_ep_incr_index(droq->read_idx, 1,
863 droq->nb_desc);
864 droq->stats.dropped_zlp++;
865 droq->refill_count++;
866 goto oq_read_fail;
867 }
868 }
869 if (next_fetch) {
870 next_idx = otx_ep_incr_index(droq->read_idx, 1, droq->nb_desc);
871 droq_pkt2 = droq->recv_buf_list[next_idx];
872 info2 = rte_pktmbuf_mtod(droq_pkt2, struct otx_ep_droq_info *);
873 rte_prefetch_non_temporal((const void *)info2);
874 }
875
876 info->length = rte_bswap64(info->length);
877 /* Deduce the actual data size */
878 total_pkt_len = info->length + INFO_SIZE;
879 if (total_pkt_len <= droq->buffer_size) {
880 info->length -= OTX_EP_RH_SIZE;
881 droq_pkt = droq->recv_buf_list[droq->read_idx];
882 if (likely(droq_pkt != NULL)) {
883 droq_pkt->data_off += OTX_EP_DROQ_INFO_SIZE;
884 /* otx_ep_dbg("OQ: pkt_len[%ld], buffer_size %d\n",
885 * (long)info->length, droq->buffer_size);
886 */
887 pkt_len = (uint32_t)info->length;
888 droq_pkt->pkt_len = pkt_len;
889 droq_pkt->data_len = pkt_len;
890 droq_pkt->port = otx_ep->port_id;
891 droq->recv_buf_list[droq->read_idx] = NULL;
892 droq->read_idx = otx_ep_incr_index(droq->read_idx, 1,
893 droq->nb_desc);
894 droq->refill_count++;
895 }
896 } else {
897 struct rte_mbuf *first_buf = NULL;
898 struct rte_mbuf *last_buf = NULL;
899
900 while (pkt_len < total_pkt_len) {
901 int cpy_len = 0;
902
903 cpy_len = ((pkt_len + droq->buffer_size) >
904 total_pkt_len)
905 ? ((uint32_t)total_pkt_len -
906 pkt_len)
907 : droq->buffer_size;
908
909 droq_pkt = droq->recv_buf_list[droq->read_idx];
910 droq->recv_buf_list[droq->read_idx] = NULL;
911
912 if (likely(droq_pkt != NULL)) {
913 /* Note the first seg */
914 if (!pkt_len)
915 first_buf = droq_pkt;
916
917 droq_pkt->port = otx_ep->port_id;
918 if (!pkt_len) {
919 droq_pkt->data_off +=
920 OTX_EP_DROQ_INFO_SIZE;
921 droq_pkt->pkt_len =
922 cpy_len - OTX_EP_DROQ_INFO_SIZE;
923 droq_pkt->data_len =
924 cpy_len - OTX_EP_DROQ_INFO_SIZE;
925 } else {
926 droq_pkt->pkt_len = cpy_len;
927 droq_pkt->data_len = cpy_len;
928 }
929
930 if (pkt_len) {
931 first_buf->nb_segs++;
932 first_buf->pkt_len += droq_pkt->pkt_len;
933 }
934
935 if (last_buf)
936 last_buf->next = droq_pkt;
937
938 last_buf = droq_pkt;
939 } else {
940 otx_ep_err("no buf\n");
941 }
942
943 pkt_len += cpy_len;
944 droq->read_idx = otx_ep_incr_index(droq->read_idx, 1,
945 droq->nb_desc);
946 droq->refill_count++;
947 }
948 droq_pkt = first_buf;
949 }
950 droq_pkt->packet_type = rte_net_get_ptype(droq_pkt, &hdr_lens,
951 RTE_PTYPE_ALL_MASK);
952 droq_pkt->l2_len = hdr_lens.l2_len;
953 droq_pkt->l3_len = hdr_lens.l3_len;
954 droq_pkt->l4_len = hdr_lens.l4_len;
955
956 if (droq_pkt->nb_segs > 1 &&
957 !(otx_ep->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER)) {
958 rte_pktmbuf_free(droq_pkt);
959 goto oq_read_fail;
960 }
961
962 return droq_pkt;
963
964 oq_read_fail:
965 return NULL;
966 }
967
968 static inline uint32_t
otx_ep_check_droq_pkts(struct otx_ep_droq * droq)969 otx_ep_check_droq_pkts(struct otx_ep_droq *droq)
970 {
971 volatile uint64_t pkt_count;
972 uint32_t new_pkts;
973
974 /* Latest available OQ packets */
975 pkt_count = rte_read32(droq->pkts_sent_reg);
976 rte_write32(pkt_count, droq->pkts_sent_reg);
977 new_pkts = pkt_count;
978 droq->pkts_pending += new_pkts;
979 return new_pkts;
980 }
981
982 /* Check for response arrival from OCTEON 9
983 * returns number of requests completed
984 */
985 uint16_t
otx_ep_recv_pkts(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t budget)986 otx_ep_recv_pkts(void *rx_queue,
987 struct rte_mbuf **rx_pkts,
988 uint16_t budget)
989 {
990 struct otx_ep_droq *droq = rx_queue;
991 struct otx_ep_device *otx_ep;
992 struct rte_mbuf *oq_pkt;
993
994 uint32_t pkts = 0;
995 uint32_t new_pkts = 0;
996 int next_fetch;
997
998 otx_ep = droq->otx_ep_dev;
999
1000 if (droq->pkts_pending > budget) {
1001 new_pkts = budget;
1002 } else {
1003 new_pkts = droq->pkts_pending;
1004 new_pkts += otx_ep_check_droq_pkts(droq);
1005 if (new_pkts > budget)
1006 new_pkts = budget;
1007 }
1008
1009 if (!new_pkts)
1010 goto update_credit; /* No pkts at this moment */
1011
1012 for (pkts = 0; pkts < new_pkts; pkts++) {
1013 /* Push the received pkt to application */
1014 next_fetch = (pkts == new_pkts - 1) ? 0 : 1;
1015 oq_pkt = otx_ep_droq_read_packet(otx_ep, droq, next_fetch);
1016 if (!oq_pkt) {
1017 RTE_LOG_DP(ERR, PMD,
1018 "DROQ read pkt failed pending %" PRIu64
1019 "last_pkt_count %" PRIu64 "new_pkts %d.\n",
1020 droq->pkts_pending, droq->last_pkt_count,
1021 new_pkts);
1022 droq->pkts_pending -= pkts;
1023 droq->stats.rx_err++;
1024 goto finish;
1025 }
1026 rx_pkts[pkts] = oq_pkt;
1027 /* Stats */
1028 droq->stats.pkts_received++;
1029 droq->stats.bytes_received += oq_pkt->pkt_len;
1030 }
1031 droq->pkts_pending -= pkts;
1032
1033 /* Refill DROQ buffers */
1034 update_credit:
1035 if (droq->refill_count >= DROQ_REFILL_THRESHOLD) {
1036 int desc_refilled = otx_ep_droq_refill(droq);
1037
1038 /* Flush the droq descriptor data to memory to be sure
1039 * that when we update the credits the data in memory is
1040 * accurate.
1041 */
1042 rte_wmb();
1043 rte_write32(desc_refilled, droq->pkts_credit_reg);
1044 } else {
1045 /*
1046 * SDP output goes into DROP state when output doorbell count
1047 * goes below drop count. When door bell count is written with
1048 * a value greater than drop count SDP output should come out
1049 * of DROP state. Due to a race condition this is not happening.
1050 * Writing doorbell register with 0 again may make SDP output
1051 * come out of this state.
1052 */
1053
1054 rte_write32(0, droq->pkts_credit_reg);
1055 }
1056 finish:
1057 return pkts;
1058 }
1059