1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018-2021 NXP
3 */
4
5 #include "pfe_logs.h"
6 #include "pfe_mod.h"
7 #include <sys/ioctl.h>
8 #include <sys/epoll.h>
9 #include <sys/eventfd.h>
10 #include <arpa/inet.h>
11
12 static int
pfe_hif_alloc_descr(struct pfe_hif * hif)13 pfe_hif_alloc_descr(struct pfe_hif *hif)
14 {
15 void *addr;
16 int err = 0;
17
18 PMD_INIT_FUNC_TRACE();
19
20 addr = rte_zmalloc(NULL, HIF_RX_DESC_NT * sizeof(struct hif_desc) +
21 HIF_TX_DESC_NT * sizeof(struct hif_desc), RTE_CACHE_LINE_SIZE);
22 if (!addr) {
23 PFE_PMD_ERR("Could not allocate buffer descriptors!");
24 err = -ENOMEM;
25 goto err0;
26 }
27
28 hif->descr_baseaddr_p = pfe_mem_vtop((uintptr_t)addr);
29 hif->descr_baseaddr_v = addr;
30 hif->rx_ring_size = HIF_RX_DESC_NT;
31 hif->tx_ring_size = HIF_TX_DESC_NT;
32
33 return 0;
34
35 err0:
36 return err;
37 }
38
39 static void
pfe_hif_free_descr(struct pfe_hif * hif)40 pfe_hif_free_descr(struct pfe_hif *hif)
41 {
42 PMD_INIT_FUNC_TRACE();
43
44 rte_free(hif->descr_baseaddr_v);
45 }
46
47 /* pfe_hif_release_buffers */
48 static void
pfe_hif_release_buffers(struct pfe_hif * hif)49 pfe_hif_release_buffers(struct pfe_hif *hif)
50 {
51 struct hif_desc *desc;
52 uint32_t i = 0;
53 struct rte_mbuf *mbuf;
54 struct rte_pktmbuf_pool_private *mb_priv;
55
56 hif->rx_base = hif->descr_baseaddr_v;
57
58 /*Free Rx buffers */
59 desc = hif->rx_base;
60 mb_priv = rte_mempool_get_priv(hif->shm->pool);
61 for (i = 0; i < hif->rx_ring_size; i++) {
62 if (readl(&desc->data)) {
63 if (i < hif->shm->rx_buf_pool_cnt &&
64 !hif->shm->rx_buf_pool[i]) {
65 mbuf = hif->rx_buf_vaddr[i] + PFE_PKT_HEADER_SZ
66 - sizeof(struct rte_mbuf)
67 - RTE_PKTMBUF_HEADROOM
68 - mb_priv->mbuf_priv_size;
69 hif->shm->rx_buf_pool[i] = mbuf;
70 }
71 }
72 writel(0, &desc->data);
73 writel(0, &desc->status);
74 writel(0, &desc->ctrl);
75 desc++;
76 }
77 }
78
79 /*
80 * pfe_hif_init_buffers
81 * This function initializes the HIF Rx/Tx ring descriptors and
82 * initialize Rx queue with buffers.
83 */
84 int
pfe_hif_init_buffers(struct pfe_hif * hif)85 pfe_hif_init_buffers(struct pfe_hif *hif)
86 {
87 struct hif_desc *desc, *first_desc_p;
88 uint32_t i = 0;
89
90 PMD_INIT_FUNC_TRACE();
91
92 /* Check enough Rx buffers available in the shared memory */
93 if (hif->shm->rx_buf_pool_cnt < hif->rx_ring_size)
94 return -ENOMEM;
95
96 hif->rx_base = hif->descr_baseaddr_v;
97 memset(hif->rx_base, 0, hif->rx_ring_size * sizeof(struct hif_desc));
98
99 /*Initialize Rx descriptors */
100 desc = hif->rx_base;
101 first_desc_p = (struct hif_desc *)hif->descr_baseaddr_p;
102
103 for (i = 0; i < hif->rx_ring_size; i++) {
104 /* Initialize Rx buffers from the shared memory */
105 struct rte_mbuf *mbuf =
106 (struct rte_mbuf *)hif->shm->rx_buf_pool[i];
107
108 /* PFE mbuf structure is as follow:
109 * ----------------------------------------------------------+
110 * | mbuf | priv | headroom (annotation + PFE data) | data |
111 * ----------------------------------------------------------+
112 *
113 * As we are expecting additional information like parse
114 * results, eth id, queue id from PFE block along with data.
115 * so we have to provide additional memory for each packet to
116 * HIF rx rings so that PFE block can write its headers.
117 * so, we are giving the data pointer to HIF rings whose
118 * calculation is as below:
119 * mbuf->data_pointer - Required_header_size
120 *
121 * We are utilizing the HEADROOM area to receive the PFE
122 * block headers. On packet reception, HIF driver will use
123 * PFE headers information based on which it will decide
124 * the clients and fill the parse results.
125 * after that application can use/overwrite the HEADROOM area.
126 */
127 hif->rx_buf_vaddr[i] =
128 (void *)((size_t)mbuf->buf_addr + mbuf->data_off -
129 PFE_PKT_HEADER_SZ);
130 hif->rx_buf_addr[i] =
131 (void *)(size_t)(rte_pktmbuf_iova(mbuf) -
132 PFE_PKT_HEADER_SZ);
133 hif->rx_buf_len[i] = mbuf->buf_len - RTE_PKTMBUF_HEADROOM;
134
135 hif->shm->rx_buf_pool[i] = NULL;
136
137 writel(DDR_PHYS_TO_PFE(hif->rx_buf_addr[i]),
138 &desc->data);
139 writel(0, &desc->status);
140
141 /*
142 * Ensure everything else is written to DDR before
143 * writing bd->ctrl
144 */
145 rte_wmb();
146
147 writel((BD_CTRL_PKT_INT_EN | BD_CTRL_LIFM
148 | BD_CTRL_DIR | BD_CTRL_DESC_EN
149 | BD_BUF_LEN(hif->rx_buf_len[i])), &desc->ctrl);
150
151 /* Chain descriptors */
152 writel((u32)DDR_PHYS_TO_PFE(first_desc_p + i + 1), &desc->next);
153 desc++;
154 }
155
156 /* Overwrite last descriptor to chain it to first one*/
157 desc--;
158 writel((u32)DDR_PHYS_TO_PFE(first_desc_p), &desc->next);
159
160 hif->rxtoclean_index = 0;
161
162 /*Initialize Rx buffer descriptor ring base address */
163 writel(DDR_PHYS_TO_PFE(hif->descr_baseaddr_p), HIF_RX_BDP_ADDR);
164
165 hif->tx_base = hif->rx_base + hif->rx_ring_size;
166 first_desc_p = (struct hif_desc *)hif->descr_baseaddr_p +
167 hif->rx_ring_size;
168 memset(hif->tx_base, 0, hif->tx_ring_size * sizeof(struct hif_desc));
169
170 /*Initialize tx descriptors */
171 desc = hif->tx_base;
172
173 for (i = 0; i < hif->tx_ring_size; i++) {
174 /* Chain descriptors */
175 writel((u32)DDR_PHYS_TO_PFE(first_desc_p + i + 1), &desc->next);
176 writel(0, &desc->ctrl);
177 desc++;
178 }
179
180 /* Overwrite last descriptor to chain it to first one */
181 desc--;
182 writel((u32)DDR_PHYS_TO_PFE(first_desc_p), &desc->next);
183 hif->txavail = hif->tx_ring_size;
184 hif->txtosend = 0;
185 hif->txtoclean = 0;
186 hif->txtoflush = 0;
187
188 /*Initialize Tx buffer descriptor ring base address */
189 writel((u32)DDR_PHYS_TO_PFE(first_desc_p), HIF_TX_BDP_ADDR);
190
191 return 0;
192 }
193
194 /*
195 * pfe_hif_client_register
196 *
197 * This function used to register a client driver with the HIF driver.
198 *
199 * Return value:
200 * 0 - on Successful registration
201 */
202 static int
pfe_hif_client_register(struct pfe_hif * hif,u32 client_id,struct hif_client_shm * client_shm)203 pfe_hif_client_register(struct pfe_hif *hif, u32 client_id,
204 struct hif_client_shm *client_shm)
205 {
206 struct hif_client *client = &hif->client[client_id];
207 u32 i, cnt;
208 struct rx_queue_desc *rx_qbase;
209 struct tx_queue_desc *tx_qbase;
210 struct hif_rx_queue *rx_queue;
211 struct hif_tx_queue *tx_queue;
212 int err = 0;
213
214 PMD_INIT_FUNC_TRACE();
215
216 rte_spinlock_lock(&hif->tx_lock);
217
218 if (test_bit(client_id, &hif->shm->g_client_status[0])) {
219 PFE_PMD_ERR("client %d already registered", client_id);
220 err = -1;
221 goto unlock;
222 }
223
224 memset(client, 0, sizeof(struct hif_client));
225
226 /* Initialize client Rx queues baseaddr, size */
227
228 cnt = CLIENT_CTRL_RX_Q_CNT(client_shm->ctrl);
229 /* Check if client is requesting for more queues than supported */
230 if (cnt > HIF_CLIENT_QUEUES_MAX)
231 cnt = HIF_CLIENT_QUEUES_MAX;
232
233 client->rx_qn = cnt;
234 rx_qbase = (struct rx_queue_desc *)client_shm->rx_qbase;
235 for (i = 0; i < cnt; i++) {
236 rx_queue = &client->rx_q[i];
237 rx_queue->base = rx_qbase + i * client_shm->rx_qsize;
238 rx_queue->size = client_shm->rx_qsize;
239 rx_queue->write_idx = 0;
240 }
241
242 /* Initialize client Tx queues baseaddr, size */
243 cnt = CLIENT_CTRL_TX_Q_CNT(client_shm->ctrl);
244
245 /* Check if client is requesting for more queues than supported */
246 if (cnt > HIF_CLIENT_QUEUES_MAX)
247 cnt = HIF_CLIENT_QUEUES_MAX;
248
249 client->tx_qn = cnt;
250 tx_qbase = (struct tx_queue_desc *)client_shm->tx_qbase;
251 for (i = 0; i < cnt; i++) {
252 tx_queue = &client->tx_q[i];
253 tx_queue->base = tx_qbase + i * client_shm->tx_qsize;
254 tx_queue->size = client_shm->tx_qsize;
255 tx_queue->ack_idx = 0;
256 }
257
258 set_bit(client_id, &hif->shm->g_client_status[0]);
259
260 unlock:
261 rte_spinlock_unlock(&hif->tx_lock);
262
263 return err;
264 }
265
266 /*
267 * pfe_hif_client_unregister
268 *
269 * This function used to unregister a client from the HIF driver.
270 *
271 */
272 static void
pfe_hif_client_unregister(struct pfe_hif * hif,u32 client_id)273 pfe_hif_client_unregister(struct pfe_hif *hif, u32 client_id)
274 {
275 PMD_INIT_FUNC_TRACE();
276
277 /*
278 * Mark client as no longer available (which prevents further packet
279 * receive for this client)
280 */
281 rte_spinlock_lock(&hif->tx_lock);
282
283 if (!test_bit(client_id, &hif->shm->g_client_status[0])) {
284 PFE_PMD_ERR("client %d not registered", client_id);
285
286 rte_spinlock_unlock(&hif->tx_lock);
287 return;
288 }
289
290 clear_bit(client_id, &hif->shm->g_client_status[0]);
291
292 rte_spinlock_unlock(&hif->tx_lock);
293 }
294
295 /*
296 * client_put_rxpacket-
297 */
298 static struct rte_mbuf *
client_put_rxpacket(struct hif_rx_queue * queue,void * pkt,u32 len,u32 flags,u32 client_ctrl,struct rte_mempool * pool,u32 * rem_len)299 client_put_rxpacket(struct hif_rx_queue *queue,
300 void *pkt, u32 len,
301 u32 flags, u32 client_ctrl,
302 struct rte_mempool *pool,
303 u32 *rem_len)
304 {
305 struct rx_queue_desc *desc = queue->base + queue->write_idx;
306 struct rte_mbuf *mbuf = NULL;
307
308
309 if (readl(&desc->ctrl) & CL_DESC_OWN) {
310 mbuf = rte_cpu_to_le_64(rte_pktmbuf_alloc(pool));
311 if (unlikely(!mbuf)) {
312 PFE_PMD_WARN("Buffer allocation failure\n");
313 return NULL;
314 }
315
316 desc->data = pkt;
317 desc->client_ctrl = client_ctrl;
318 /*
319 * Ensure everything else is written to DDR before
320 * writing bd->ctrl
321 */
322 rte_wmb();
323 writel(CL_DESC_BUF_LEN(len) | flags, &desc->ctrl);
324 queue->write_idx = (queue->write_idx + 1)
325 & (queue->size - 1);
326
327 *rem_len = mbuf->buf_len;
328 }
329
330 return mbuf;
331 }
332
333 /*
334 * pfe_hif_rx_process-
335 * This function does pfe hif rx queue processing.
336 * Dequeue packet from Rx queue and send it to corresponding client queue
337 */
338 int
pfe_hif_rx_process(struct pfe * pfe,int budget)339 pfe_hif_rx_process(struct pfe *pfe, int budget)
340 {
341 struct hif_desc *desc;
342 struct hif_hdr *pkt_hdr;
343 struct __hif_hdr hif_hdr;
344 void *free_buf;
345 int rtc, len, rx_processed = 0;
346 struct __hif_desc local_desc;
347 int flags = 0, wait_for_last = 0, retry = 0;
348 unsigned int buf_size = 0;
349 struct rte_mbuf *mbuf = NULL;
350 struct pfe_hif *hif = &pfe->hif;
351
352 rte_spinlock_lock(&hif->lock);
353
354 rtc = hif->rxtoclean_index;
355
356 while (rx_processed < budget) {
357 desc = hif->rx_base + rtc;
358
359 __memcpy12(&local_desc, desc);
360
361 /* ACK pending Rx interrupt */
362 if (local_desc.ctrl & BD_CTRL_DESC_EN) {
363 if (unlikely(wait_for_last))
364 continue;
365 else
366 break;
367 }
368
369 len = BD_BUF_LEN(local_desc.ctrl);
370 pkt_hdr = (struct hif_hdr *)hif->rx_buf_vaddr[rtc];
371
372 /* Track last HIF header received */
373 if (!hif->started) {
374 hif->started = 1;
375
376 __memcpy8(&hif_hdr, pkt_hdr);
377
378 hif->qno = hif_hdr.hdr.q_num;
379 hif->client_id = hif_hdr.hdr.client_id;
380 hif->client_ctrl = (hif_hdr.hdr.client_ctrl1 << 16) |
381 hif_hdr.hdr.client_ctrl;
382 flags = CL_DESC_FIRST;
383
384 } else {
385 flags = 0;
386 }
387
388 if (local_desc.ctrl & BD_CTRL_LIFM) {
389 flags |= CL_DESC_LAST;
390 wait_for_last = 0;
391 } else {
392 wait_for_last = 1;
393 }
394
395 /* Check for valid client id and still registered */
396 if (hif->client_id >= HIF_CLIENTS_MAX ||
397 !(test_bit(hif->client_id,
398 &hif->shm->g_client_status[0]))) {
399 PFE_PMD_INFO("packet with invalid client id %d qnum %d",
400 hif->client_id, hif->qno);
401
402 free_buf = hif->rx_buf_addr[rtc];
403
404 goto pkt_drop;
405 }
406
407 /* Check to valid queue number */
408 if (hif->client[hif->client_id].rx_qn <= hif->qno) {
409 PFE_DP_LOG(DEBUG, "packet with invalid queue: %d",
410 hif->qno);
411 hif->qno = 0;
412 }
413
414 retry:
415 mbuf =
416 client_put_rxpacket(&hif->client[hif->client_id].rx_q[hif->qno],
417 (void *)pkt_hdr, len, flags,
418 hif->client_ctrl, hif->shm->pool,
419 &buf_size);
420
421 if (unlikely(!mbuf)) {
422 if (!retry) {
423 pfe_tx_do_cleanup(pfe);
424 retry = 1;
425 goto retry;
426 }
427 rx_processed = budget;
428
429 if (flags & CL_DESC_FIRST)
430 hif->started = 0;
431
432 PFE_DP_LOG(DEBUG, "No buffers");
433 break;
434 }
435
436 retry = 0;
437
438 free_buf = (void *)(size_t)rte_pktmbuf_iova(mbuf);
439 free_buf = free_buf - PFE_PKT_HEADER_SZ;
440
441 /*Fill free buffer in the descriptor */
442 hif->rx_buf_addr[rtc] = free_buf;
443 hif->rx_buf_vaddr[rtc] = (void *)((size_t)mbuf->buf_addr +
444 mbuf->data_off - PFE_PKT_HEADER_SZ);
445 hif->rx_buf_len[rtc] = buf_size - RTE_PKTMBUF_HEADROOM;
446
447 pkt_drop:
448 writel(DDR_PHYS_TO_PFE(free_buf), &desc->data);
449 /*
450 * Ensure everything else is written to DDR before
451 * writing bd->ctrl
452 */
453 rte_wmb();
454 writel((BD_CTRL_PKT_INT_EN | BD_CTRL_LIFM | BD_CTRL_DIR |
455 BD_CTRL_DESC_EN | BD_BUF_LEN(hif->rx_buf_len[rtc])),
456 &desc->ctrl);
457
458 rtc = (rtc + 1) & (hif->rx_ring_size - 1);
459
460 if (local_desc.ctrl & BD_CTRL_LIFM) {
461 if (!(hif->client_ctrl & HIF_CTRL_RX_CONTINUED))
462 rx_processed++;
463
464 hif->started = 0;
465 }
466 }
467
468
469 hif->rxtoclean_index = rtc;
470 rte_spinlock_unlock(&hif->lock);
471
472 /* we made some progress, re-start rx dma in case it stopped */
473 hif_rx_dma_start();
474
475 return rx_processed;
476 }
477
478 /*
479 * client_ack_txpacket-
480 * This function ack the Tx packet in the give client Tx queue by resetting
481 * ownership bit in the descriptor.
482 */
483 static int
client_ack_txpacket(struct pfe_hif * hif,unsigned int client_id,unsigned int q_no)484 client_ack_txpacket(struct pfe_hif *hif, unsigned int client_id,
485 unsigned int q_no)
486 {
487 struct hif_tx_queue *queue = &hif->client[client_id].tx_q[q_no];
488 struct tx_queue_desc *desc = queue->base + queue->ack_idx;
489
490 if (readl(&desc->ctrl) & CL_DESC_OWN) {
491 writel((readl(&desc->ctrl) & ~CL_DESC_OWN), &desc->ctrl);
492 queue->ack_idx = (queue->ack_idx + 1) & (queue->size - 1);
493
494 return 0;
495
496 } else {
497 /*This should not happen */
498 PFE_PMD_ERR("%d %d %d %d %d %p %d",
499 hif->txtosend, hif->txtoclean, hif->txavail,
500 client_id, q_no, queue, queue->ack_idx);
501 return 1;
502 }
503 }
504
505 static void
__hif_tx_done_process(struct pfe * pfe,int count)506 __hif_tx_done_process(struct pfe *pfe, int count)
507 {
508 struct hif_desc *desc;
509 struct hif_desc_sw *desc_sw;
510 unsigned int ttc, tx_avl;
511 int pkts_done[HIF_CLIENTS_MAX] = {0, 0};
512 struct pfe_hif *hif = &pfe->hif;
513
514 ttc = hif->txtoclean;
515 tx_avl = hif->txavail;
516
517 while ((tx_avl < hif->tx_ring_size) && count--) {
518 desc = hif->tx_base + ttc;
519
520 if (readl(&desc->ctrl) & BD_CTRL_DESC_EN)
521 break;
522
523 desc_sw = &hif->tx_sw_queue[ttc];
524
525 if (desc_sw->client_id > HIF_CLIENTS_MAX)
526 PFE_PMD_ERR("Invalid cl id %d", desc_sw->client_id);
527
528 pkts_done[desc_sw->client_id]++;
529
530 client_ack_txpacket(hif, desc_sw->client_id, desc_sw->q_no);
531
532 ttc = (ttc + 1) & (hif->tx_ring_size - 1);
533 tx_avl++;
534 }
535
536 if (pkts_done[0])
537 hif_lib_indicate_client(pfe->hif_client[0], EVENT_TXDONE_IND,
538 0);
539 if (pkts_done[1])
540 hif_lib_indicate_client(pfe->hif_client[1], EVENT_TXDONE_IND,
541 0);
542 hif->txtoclean = ttc;
543 hif->txavail = tx_avl;
544 }
545
546 static inline void
hif_tx_done_process(struct pfe * pfe,int count)547 hif_tx_done_process(struct pfe *pfe, int count)
548 {
549 struct pfe_hif *hif = &pfe->hif;
550 rte_spinlock_lock(&hif->tx_lock);
551 __hif_tx_done_process(pfe, count);
552 rte_spinlock_unlock(&hif->tx_lock);
553 }
554
555 void
pfe_tx_do_cleanup(struct pfe * pfe)556 pfe_tx_do_cleanup(struct pfe *pfe)
557 {
558 hif_tx_done_process(pfe, HIF_TX_DESC_NT);
559 }
560
561 /*
562 * __hif_xmit_pkt -
563 * This function puts one packet in the HIF Tx queue
564 */
565 void
hif_xmit_pkt(struct pfe_hif * hif,unsigned int client_id,unsigned int q_no,void * data,u32 len,unsigned int flags)566 hif_xmit_pkt(struct pfe_hif *hif, unsigned int client_id, unsigned int
567 q_no, void *data, u32 len, unsigned int flags)
568 {
569 struct hif_desc *desc;
570 struct hif_desc_sw *desc_sw;
571
572 desc = hif->tx_base + hif->txtosend;
573 desc_sw = &hif->tx_sw_queue[hif->txtosend];
574
575 desc_sw->len = len;
576 desc_sw->client_id = client_id;
577 desc_sw->q_no = q_no;
578 desc_sw->flags = flags;
579
580 writel((u32)DDR_PHYS_TO_PFE(data), &desc->data);
581
582 hif->txtosend = (hif->txtosend + 1) & (hif->tx_ring_size - 1);
583 hif->txavail--;
584
585 if ((!((flags & HIF_DATA_VALID) && (flags &
586 HIF_LAST_BUFFER))))
587 goto skip_tx;
588
589 /*
590 * Ensure everything else is written to DDR before
591 * writing bd->ctrl
592 */
593 rte_wmb();
594
595 do {
596 desc_sw = &hif->tx_sw_queue[hif->txtoflush];
597 desc = hif->tx_base + hif->txtoflush;
598
599 if (desc_sw->flags & HIF_LAST_BUFFER) {
600 writel((BD_CTRL_LIFM |
601 BD_CTRL_BRFETCH_DISABLE | BD_CTRL_RTFETCH_DISABLE
602 | BD_CTRL_PARSE_DISABLE | BD_CTRL_DESC_EN |
603 BD_BUF_LEN(desc_sw->len)),
604 &desc->ctrl);
605 } else {
606 writel((BD_CTRL_DESC_EN |
607 BD_BUF_LEN(desc_sw->len)), &desc->ctrl);
608 }
609 hif->txtoflush = (hif->txtoflush + 1) & (hif->tx_ring_size - 1);
610 }
611 while (hif->txtoflush != hif->txtosend)
612 ;
613
614 skip_tx:
615 return;
616 }
617
618 void
hif_process_client_req(struct pfe_hif * hif,int req,int data1,__rte_unused int data2)619 hif_process_client_req(struct pfe_hif *hif, int req,
620 int data1, __rte_unused int data2)
621 {
622 unsigned int client_id = data1;
623
624 if (client_id >= HIF_CLIENTS_MAX) {
625 PFE_PMD_ERR("client id %d out of bounds", client_id);
626 return;
627 }
628
629 switch (req) {
630 case REQUEST_CL_REGISTER:
631 /* Request for register a client */
632 PFE_PMD_INFO("register client_id %d", client_id);
633 pfe_hif_client_register(hif, client_id, (struct
634 hif_client_shm *)&hif->shm->client[client_id]);
635 break;
636
637 case REQUEST_CL_UNREGISTER:
638 PFE_PMD_INFO("unregister client_id %d", client_id);
639
640 /* Request for unregister a client */
641 pfe_hif_client_unregister(hif, client_id);
642
643 break;
644
645 default:
646 PFE_PMD_ERR("unsupported request %d", req);
647 break;
648 }
649
650 /*
651 * Process client Tx queues
652 * Currently we don't have checking for tx pending
653 */
654 }
655
656 #if defined(LS1012A_PFE_RESET_WA)
657 static void
pfe_hif_disable_rx_desc(struct pfe_hif * hif)658 pfe_hif_disable_rx_desc(struct pfe_hif *hif)
659 {
660 u32 ii;
661 struct hif_desc *desc = hif->rx_base;
662
663 /*Mark all descriptors as LAST_BD */
664 for (ii = 0; ii < hif->rx_ring_size; ii++) {
665 desc->ctrl |= BD_CTRL_LAST_BD;
666 desc++;
667 }
668 }
669
670 struct class_rx_hdr_t {
671 u32 next_ptr; /* ptr to the start of the first DDR buffer */
672 u16 length; /* total packet length */
673 u16 phyno; /* input physical port number */
674 u32 status; /* gemac status bits */
675 u32 status2; /* reserved for software usage */
676 };
677
678 /* STATUS_BAD_FRAME_ERR is set for all errors (including checksums if enabled)
679 * except overflow
680 */
681 #define STATUS_BAD_FRAME_ERR BIT(16)
682 #define STATUS_LENGTH_ERR BIT(17)
683 #define STATUS_CRC_ERR BIT(18)
684 #define STATUS_TOO_SHORT_ERR BIT(19)
685 #define STATUS_TOO_LONG_ERR BIT(20)
686 #define STATUS_CODE_ERR BIT(21)
687 #define STATUS_MC_HASH_MATCH BIT(22)
688 #define STATUS_CUMULATIVE_ARC_HIT BIT(23)
689 #define STATUS_UNICAST_HASH_MATCH BIT(24)
690 #define STATUS_IP_CHECKSUM_CORRECT BIT(25)
691 #define STATUS_TCP_CHECKSUM_CORRECT BIT(26)
692 #define STATUS_UDP_CHECKSUM_CORRECT BIT(27)
693 #define STATUS_OVERFLOW_ERR BIT(28) /* GPI error */
694 #define MIN_PKT_SIZE 64
695 #define DUMMY_PKT_COUNT 128
696
697 static inline void
copy_to_lmem(u32 * dst,u32 * src,int len)698 copy_to_lmem(u32 *dst, u32 *src, int len)
699 {
700 int i;
701
702 for (i = 0; i < len; i += sizeof(u32)) {
703 *dst = htonl(*src);
704 dst++; src++;
705 }
706 }
707 #if defined(RTE_TOOLCHAIN_GCC)
708 __attribute__ ((optimize(1)))
709 #endif
710 static void
send_dummy_pkt_to_hif(void)711 send_dummy_pkt_to_hif(void)
712 {
713 void *lmem_ptr, *ddr_ptr, *lmem_virt_addr;
714 u64 physaddr;
715 struct class_rx_hdr_t local_hdr;
716 static u32 dummy_pkt[] = {
717 0x33221100, 0x2b785544, 0xd73093cb, 0x01000608,
718 0x04060008, 0x2b780200, 0xd73093cb, 0x0a01a8c0,
719 0x33221100, 0xa8c05544, 0x00000301, 0x00000000,
720 0x00000000, 0x00000000, 0x00000000, 0xbe86c51f };
721
722 ddr_ptr = (void *)(size_t)readl(BMU2_BASE_ADDR + BMU_ALLOC_CTRL);
723 if (!ddr_ptr)
724 return;
725
726 lmem_ptr = (void *)(size_t)readl(BMU1_BASE_ADDR + BMU_ALLOC_CTRL);
727 if (!lmem_ptr)
728 return;
729
730 PFE_PMD_INFO("Sending a dummy pkt to HIF %p %p", ddr_ptr, lmem_ptr);
731 physaddr = DDR_VIRT_TO_PFE(ddr_ptr);
732
733 lmem_virt_addr = (void *)CBUS_PFE_TO_VIRT((unsigned long)lmem_ptr);
734
735 local_hdr.phyno = htons(0); /* RX_PHY_0 */
736 local_hdr.length = htons(MIN_PKT_SIZE);
737
738 local_hdr.next_ptr = htonl((u32)physaddr);
739 /*Mark checksum is correct */
740 local_hdr.status = htonl((STATUS_IP_CHECKSUM_CORRECT |
741 STATUS_UDP_CHECKSUM_CORRECT |
742 STATUS_TCP_CHECKSUM_CORRECT |
743 STATUS_UNICAST_HASH_MATCH |
744 STATUS_CUMULATIVE_ARC_HIT));
745 copy_to_lmem((u32 *)lmem_virt_addr, (u32 *)&local_hdr,
746 sizeof(local_hdr));
747
748 copy_to_lmem((u32 *)(lmem_virt_addr + LMEM_HDR_SIZE), (u32 *)dummy_pkt,
749 0x40);
750
751 writel((unsigned long)lmem_ptr, CLASS_INQ_PKTPTR);
752 }
753
754 void
pfe_hif_rx_idle(struct pfe_hif * hif)755 pfe_hif_rx_idle(struct pfe_hif *hif)
756 {
757 int hif_stop_loop = DUMMY_PKT_COUNT;
758 u32 rx_status;
759
760 pfe_hif_disable_rx_desc(hif);
761 PFE_PMD_INFO("Bringing hif to idle state...");
762 writel(0, HIF_INT_ENABLE);
763 /*If HIF Rx BDP is busy send a dummy packet */
764 do {
765 rx_status = readl(HIF_RX_STATUS);
766 if (rx_status & BDP_CSR_RX_DMA_ACTV)
767 send_dummy_pkt_to_hif();
768
769 rte_delay_ms(1);
770 } while (--hif_stop_loop);
771
772 if (readl(HIF_RX_STATUS) & BDP_CSR_RX_DMA_ACTV)
773 PFE_PMD_ERR("Failed\n");
774 else
775 PFE_PMD_INFO("Done\n");
776 }
777 #endif
778
779 /*
780 * pfe_hif_init
781 * This function initializes the baseaddresses and irq, etc.
782 */
783 int
pfe_hif_init(struct pfe * pfe)784 pfe_hif_init(struct pfe *pfe)
785 {
786 struct pfe_hif *hif = &pfe->hif;
787 int err;
788
789 PMD_INIT_FUNC_TRACE();
790
791 #if defined(LS1012A_PFE_RESET_WA)
792 pfe_hif_rx_idle(hif);
793 #endif
794
795 err = pfe_hif_alloc_descr(hif);
796 if (err)
797 goto err0;
798
799 rte_spinlock_init(&hif->tx_lock);
800 rte_spinlock_init(&hif->lock);
801
802 gpi_enable(HGPI_BASE_ADDR);
803 if (getenv("PFE_INTR_SUPPORT")) {
804 struct epoll_event epoll_ev;
805 int event_fd = -1, epoll_fd, pfe_cdev_fd;
806
807 pfe_cdev_fd = open(PFE_CDEV_PATH, O_RDWR);
808 if (pfe_cdev_fd < 0) {
809 PFE_PMD_WARN("Unable to open PFE device file (%s).\n",
810 PFE_CDEV_PATH);
811 pfe->cdev_fd = PFE_CDEV_INVALID_FD;
812 return -1;
813 }
814 pfe->cdev_fd = pfe_cdev_fd;
815
816 event_fd = eventfd(0, EFD_NONBLOCK);
817 /* hif interrupt enable */
818 err = ioctl(pfe->cdev_fd, PFE_CDEV_HIF_INTR_EN, &event_fd);
819 if (err) {
820 PFE_PMD_ERR("\nioctl failed for intr enable err: %d\n",
821 errno);
822 goto err0;
823 }
824 epoll_fd = epoll_create(1);
825 epoll_ev.events = EPOLLIN | EPOLLPRI | EPOLLET;
826 epoll_ev.data.fd = event_fd;
827 err = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, event_fd, &epoll_ev);
828 if (err < 0) {
829 PFE_PMD_ERR("epoll_ctl failed with err = %d\n", errno);
830 goto err0;
831 }
832 pfe->hif.epoll_fd = epoll_fd;
833 }
834 return 0;
835 err0:
836 return err;
837 }
838
839 /* pfe_hif_exit- */
840 void
pfe_hif_exit(struct pfe * pfe)841 pfe_hif_exit(struct pfe *pfe)
842 {
843 struct pfe_hif *hif = &pfe->hif;
844
845 PMD_INIT_FUNC_TRACE();
846
847 rte_spinlock_lock(&hif->lock);
848 hif->shm->g_client_status[0] = 0;
849 /* Make sure all clients are disabled*/
850 hif->shm->g_client_status[1] = 0;
851
852 rte_spinlock_unlock(&hif->lock);
853
854 if (hif->setuped) {
855 #if defined(LS1012A_PFE_RESET_WA)
856 pfe_hif_rx_idle(hif);
857 #endif
858 /*Disable Rx/Tx */
859 hif_rx_disable();
860 hif_tx_disable();
861
862 pfe_hif_release_buffers(hif);
863 pfe_hif_shm_clean(hif->shm);
864
865 pfe_hif_free_descr(hif);
866 pfe->hif.setuped = 0;
867 }
868 gpi_disable(HGPI_BASE_ADDR);
869 }
870