| /dpdk/drivers/net/i40e/ |
| H A D | i40e_rxtx_vec_avx2.c | 726 vtx1(txdp, *pkt, flags); in vtx()
727 nb_pkts--, txdp++, pkt++; in vtx()
753 vtx1(txdp, *pkt, flags); in vtx()
754 txdp++, pkt++, nb_pkts--; in vtx()
763 volatile struct i40e_tx_desc *txdp; in i40e_xmit_fixed_burst_vec_avx2() local
777 txdp = &txq->tx_ring[tx_id]; in i40e_xmit_fixed_burst_vec_avx2()
786 vtx(txdp, tx_pkts, n - 1, flags); in i40e_xmit_fixed_burst_vec_avx2()
788 txdp += (n - 1); in i40e_xmit_fixed_burst_vec_avx2()
790 vtx1(txdp, *tx_pkts++, rs); in i40e_xmit_fixed_burst_vec_avx2()
798 txdp = &txq->tx_ring[tx_id]; in i40e_xmit_fixed_burst_vec_avx2()
[all …]
|
| H A D | i40e_rxtx_vec_altivec.c | 529 vtx1(volatile struct i40e_tx_desc *txdp, in vtx1() argument
538 *(__vector unsigned long *)txdp = descriptor; in vtx1()
542 vtx(volatile struct i40e_tx_desc *txdp, in vtx() argument
547 for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) in vtx()
548 vtx1(txdp, *pkt, flags); in vtx()
556 volatile struct i40e_tx_desc *txdp; in i40e_xmit_fixed_burst_vec() local
572 txdp = &txq->tx_ring[tx_id]; in i40e_xmit_fixed_burst_vec()
582 vtx1(txdp, *tx_pkts, flags); in i40e_xmit_fixed_burst_vec()
584 vtx1(txdp, *tx_pkts++, rs); in i40e_xmit_fixed_burst_vec()
592 txdp = &txq->tx_ring[tx_id]; in i40e_xmit_fixed_burst_vec()
[all …]
|
| H A D | i40e_rxtx_vec_sse.c | 678 vtx1(volatile struct i40e_tx_desc *txdp, in vtx1() argument
687 _mm_store_si128((__m128i *)txdp, descriptor); in vtx1()
691 vtx(volatile struct i40e_tx_desc *txdp, in vtx() argument
696 for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) in vtx()
697 vtx1(txdp, *pkt, flags); in vtx()
705 volatile struct i40e_tx_desc *txdp; in i40e_xmit_fixed_burst_vec() local
720 txdp = &txq->tx_ring[tx_id]; in i40e_xmit_fixed_burst_vec()
730 vtx1(txdp, *tx_pkts, flags); in i40e_xmit_fixed_burst_vec()
732 vtx1(txdp, *tx_pkts++, rs); in i40e_xmit_fixed_burst_vec()
740 txdp = &txq->tx_ring[tx_id]; in i40e_xmit_fixed_burst_vec()
[all …]
|
| H A D | i40e_rxtx_vec_neon.c | 656 vtx1(volatile struct i40e_tx_desc *txdp, in vtx1() argument
664 vst1q_u64((uint64_t *)txdp, descriptor); in vtx1()
673 for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) in vtx()
674 vtx1(txdp, *pkt, flags); in vtx()
682 volatile struct i40e_tx_desc *txdp; in i40e_xmit_fixed_burst_vec() local
697 txdp = &txq->tx_ring[tx_id]; in i40e_xmit_fixed_burst_vec()
706 for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) in i40e_xmit_fixed_burst_vec()
707 vtx1(txdp, *tx_pkts, flags); in i40e_xmit_fixed_burst_vec()
709 vtx1(txdp, *tx_pkts++, rs); in i40e_xmit_fixed_burst_vec()
717 txdp = &txq->tx_ring[tx_id]; in i40e_xmit_fixed_burst_vec()
[all …]
|
| H A D | i40e_rxtx_vec_avx512.c | 1002 vtx(volatile struct i40e_tx_desc *txdp, in vtx() argument
1032 _mm512_storeu_si512((void *)txdp, desc0_3); in vtx()
1037 vtx1(txdp, *pkt, flags); in vtx()
1038 txdp++, pkt++, nb_pkts--; in vtx()
1057 volatile struct i40e_tx_desc *txdp; in i40e_xmit_fixed_burst_vec_avx512() local
1071 txdp = &txq->tx_ring[tx_id]; in i40e_xmit_fixed_burst_vec_avx512()
1081 vtx(txdp, tx_pkts, n - 1, flags); in i40e_xmit_fixed_burst_vec_avx512()
1083 txdp += (n - 1); in i40e_xmit_fixed_burst_vec_avx512()
1085 vtx1(txdp, *tx_pkts++, rs); in i40e_xmit_fixed_burst_vec_avx512()
1093 txdp = txq->tx_ring; in i40e_xmit_fixed_burst_vec_avx512()
[all …]
|
| H A D | i40e_rxtx.c | 1378 tx4(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts) in tx4() argument
1383 for (i = 0; i < 4; i++, txdp++, pkts++) { in tx4()
1385 txdp->buffer_addr = rte_cpu_to_le_64(dma_addr); in tx4()
1386 txdp->cmd_type_offset_bsz = in tx4()
1394 tx1(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts) in tx1() argument
1399 txdp->buffer_addr = rte_cpu_to_le_64(dma_addr); in tx1()
1400 txdp->cmd_type_offset_bsz = in tx1()
1411 volatile struct i40e_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); in i40e_tx_fill_hw_ring() local
1424 tx4(txdp + i, pkts + i); in i40e_tx_fill_hw_ring()
1429 tx1(txdp + mainpart + i, pkts + mainpart + i); in i40e_tx_fill_hw_ring()
|
| H A D | i40e_fdir.c | 1933 volatile struct i40e_tx_desc *txdp; in i40e_flow_fdir_filter_programming() local
2012 txdp = &txq->tx_ring[txq->tx_tail + 1]; in i40e_flow_fdir_filter_programming()
2013 txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr[txq->tx_tail >> 1]); in i40e_flow_fdir_filter_programming()
2019 txdp->cmd_type_offset_bsz = in i40e_flow_fdir_filter_programming()
2035 if ((txdp->cmd_type_offset_bsz & in i40e_flow_fdir_filter_programming()
|
| /dpdk/drivers/net/ice/ |
| H A D | ice_rxtx_vec_avx2.c | 814 ice_vtx1(volatile struct ice_tx_desc *txdp, in ice_vtx1() argument
830 ice_vtx(volatile struct ice_tx_desc *txdp, in ice_vtx() argument
838 ice_vtx1(txdp, *pkt, flags, offload); in ice_vtx()
839 nb_pkts--, txdp++, pkt++; in ice_vtx()
887 ice_vtx1(txdp, *pkt, flags, offload); in ice_vtx()
888 txdp++, pkt++, nb_pkts--; in ice_vtx()
897 volatile struct ice_tx_desc *txdp; in ice_xmit_fixed_burst_vec_avx2() local
914 txdp = &txq->tx_ring[tx_id]; in ice_xmit_fixed_burst_vec_avx2()
925 txdp += (n - 1); in ice_xmit_fixed_burst_vec_avx2()
927 ice_vtx1(txdp, *tx_pkts++, rs, offload); in ice_xmit_fixed_burst_vec_avx2()
[all …]
|
| H A D | ice_rxtx_vec_sse.c | 666 ice_vtx1(volatile struct ice_tx_desc *txdp, struct rte_mbuf *pkt, in ice_vtx1() argument
676 _mm_store_si128((__m128i *)txdp, descriptor); in ice_vtx1()
685 for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) in ice_vtx()
686 ice_vtx1(txdp, *pkt, flags); in ice_vtx()
694 volatile struct ice_tx_desc *txdp; in ice_xmit_fixed_burst_vec() local
713 txdp = &txq->tx_ring[tx_id]; in ice_xmit_fixed_burst_vec()
722 for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) in ice_xmit_fixed_burst_vec()
723 ice_vtx1(txdp, *tx_pkts, flags); in ice_xmit_fixed_burst_vec()
725 ice_vtx1(txdp, *tx_pkts++, rs); in ice_xmit_fixed_burst_vec()
733 txdp = &txq->tx_ring[tx_id]; in ice_xmit_fixed_burst_vec()
[all …]
|
| H A D | ice_rxtx_vec_avx512.c | 1084 ice_vtx1(volatile struct ice_tx_desc *txdp, in ice_vtx1() argument
1097 _mm_store_si128((__m128i *)txdp, descriptor); in ice_vtx1()
1143 _mm512_storeu_si512((void *)txdp, desc0_3); in ice_vtx()
1148 ice_vtx1(txdp, *pkt, flags, do_offload); in ice_vtx()
1149 txdp++, pkt++, nb_pkts--; in ice_vtx()
1168 volatile struct ice_tx_desc *txdp; in ice_xmit_fixed_burst_vec_avx512() local
1185 txdp = &txq->tx_ring[tx_id]; in ice_xmit_fixed_burst_vec_avx512()
1195 ice_vtx(txdp, tx_pkts, n - 1, flags, do_offload); in ice_xmit_fixed_burst_vec_avx512()
1197 txdp += (n - 1); in ice_xmit_fixed_burst_vec_avx512()
1199 ice_vtx1(txdp, *tx_pkts++, rs, do_offload); in ice_xmit_fixed_burst_vec_avx512()
[all …]
|
| H A D | ice_rxtx.c | 3093 for (i = 0; i < 4; i++, txdp++, pkts++) { in tx4()
3095 txdp->buf_addr = rte_cpu_to_le_64(dma_addr); in tx4()
3096 txdp->cmd_type_offset_bsz = in tx4()
3109 txdp->buf_addr = rte_cpu_to_le_64(dma_addr); in tx1()
3110 txdp->cmd_type_offset_bsz = in tx1()
3136 tx4(txdp + i, pkts + i); in ice_tx_fill_hw_ring()
3142 tx1(txdp + mainpart + i, pkts + mainpart + i); in ice_tx_fill_hw_ring()
4307 volatile struct ice_tx_desc *txdp; in ice_fdir_programming() local
4316 txdp = &txq->tx_ring[txq->tx_tail + 1]; in ice_fdir_programming()
4322 txdp->cmd_type_offset_bsz = in ice_fdir_programming()
[all …]
|
| /dpdk/drivers/net/ixgbe/ |
| H A D | ixgbe_rxtx_vec_neon.c | 550 vtx1(volatile union ixgbe_adv_tx_desc *txdp, in vtx1() argument
557 vst1q_u64((uint64_t *)&txdp->read, descriptor); in vtx1()
561 vtx(volatile union ixgbe_adv_tx_desc *txdp, in vtx() argument
566 for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) in vtx()
567 vtx1(txdp, *pkt, flags); in vtx()
575 volatile union ixgbe_adv_tx_desc *txdp; in ixgbe_xmit_fixed_burst_vec() local
593 txdp = &txq->tx_ring[tx_id]; in ixgbe_xmit_fixed_burst_vec()
603 vtx1(txdp, *tx_pkts, flags); in ixgbe_xmit_fixed_burst_vec()
605 vtx1(txdp, *tx_pkts++, rs); in ixgbe_xmit_fixed_burst_vec()
613 txdp = &txq->tx_ring[tx_id]; in ixgbe_xmit_fixed_burst_vec()
[all …]
|
| H A D | ixgbe_rxtx_vec_sse.c | 673 vtx1(volatile union ixgbe_adv_tx_desc *txdp, in vtx1() argument
683 vtx(volatile union ixgbe_adv_tx_desc *txdp, in vtx() argument
688 for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) in vtx()
689 vtx1(txdp, *pkt, flags); in vtx()
697 volatile union ixgbe_adv_tx_desc *txdp; in ixgbe_xmit_fixed_burst_vec() local
715 txdp = &txq->tx_ring[tx_id]; in ixgbe_xmit_fixed_burst_vec()
725 for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) in ixgbe_xmit_fixed_burst_vec()
726 vtx1(txdp, *tx_pkts, flags); in ixgbe_xmit_fixed_burst_vec()
728 vtx1(txdp, *tx_pkts++, rs); in ixgbe_xmit_fixed_burst_vec()
736 txdp = &(txq->tx_ring[tx_id]); in ixgbe_xmit_fixed_burst_vec()
[all …]
|
| H A D | ixgbe_rxtx.c | 152 tx4(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts) in tx4() argument
158 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { in tx4()
163 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr); in tx4()
165 txdp->read.cmd_type_len = in tx4()
168 txdp->read.olinfo_status = in tx4()
177 tx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts) in tx1() argument
186 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr); in tx1()
187 txdp->read.cmd_type_len = in tx1()
189 txdp->read.olinfo_status = in tx1()
220 tx4(txdp + i, pkts + i); in ixgbe_tx_fill_hw_ring()
[all …]
|
| /dpdk/drivers/net/iavf/ |
| H A D | iavf_rxtx_vec_avx2.c | 1395 iavf_vtx1(txdp, *pkt, flags); in iavf_vtx()
1396 nb_pkts--, txdp++, pkt++; in iavf_vtx()
1436 iavf_vtx1(txdp, *pkt, flags); in iavf_vtx()
1437 txdp++, pkt++, nb_pkts--; in iavf_vtx()
1446 volatile struct iavf_tx_desc *txdp; in iavf_xmit_fixed_burst_vec_avx2() local
1461 txdp = &txq->tx_ring[tx_id]; in iavf_xmit_fixed_burst_vec_avx2()
1470 iavf_vtx(txdp, tx_pkts, n - 1, flags); in iavf_xmit_fixed_burst_vec_avx2()
1472 txdp += (n - 1); in iavf_xmit_fixed_burst_vec_avx2()
1474 iavf_vtx1(txdp, *tx_pkts++, rs); in iavf_xmit_fixed_burst_vec_avx2()
1482 txdp = &txq->tx_ring[tx_id]; in iavf_xmit_fixed_burst_vec_avx2()
[all …]
|
| H A D | iavf_rxtx_vec_avx512.c | 1820 iavf_vtx1(volatile struct iavf_tx_desc *txdp, in iavf_vtx1() argument
1838 iavf_vtx(volatile struct iavf_tx_desc *txdp, in iavf_vtx() argument
1847 iavf_vtx1(txdp, *pkt, flags, offload); in iavf_vtx()
1848 nb_pkts--, txdp++, pkt++; in iavf_vtx()
1893 iavf_vtx1(txdp, *pkt, flags, offload); in iavf_vtx()
1894 txdp++, pkt++, nb_pkts--; in iavf_vtx()
1903 volatile struct iavf_tx_desc *txdp; in iavf_xmit_fixed_burst_vec_avx512() local
1918 txdp = &txq->tx_ring[tx_id]; in iavf_xmit_fixed_burst_vec_avx512()
1930 txdp += (n - 1); in iavf_xmit_fixed_burst_vec_avx512()
1932 iavf_vtx1(txdp, *tx_pkts++, rs, offload); in iavf_xmit_fixed_burst_vec_avx512()
[all …]
|
| H A D | iavf_rxtx_vec_sse.c | 1098 _mm_store_si128((__m128i *)txdp, descriptor); in vtx1()
1102 iavf_vtx(volatile struct iavf_tx_desc *txdp, struct rte_mbuf **pkt, in iavf_vtx() argument
1107 for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) in iavf_vtx()
1108 vtx1(txdp, *pkt, flags); in iavf_vtx()
1116 volatile struct iavf_tx_desc *txdp; in iavf_xmit_fixed_burst_vec() local
1132 txdp = &txq->tx_ring[tx_id]; in iavf_xmit_fixed_burst_vec()
1141 for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) in iavf_xmit_fixed_burst_vec()
1142 vtx1(txdp, *tx_pkts, flags); in iavf_xmit_fixed_burst_vec()
1144 vtx1(txdp, *tx_pkts++, rs); in iavf_xmit_fixed_burst_vec()
1152 txdp = &txq->tx_ring[tx_id]; in iavf_xmit_fixed_burst_vec()
[all …]
|
| /dpdk/drivers/net/fm10k/ |
| H A D | fm10k_rxtx_vec.c | 742 vtx1(volatile struct fm10k_tx_desc *txdp, in vtx1() argument
748 _mm_store_si128((__m128i *)txdp, descriptor); in vtx1()
752 vtx(volatile struct fm10k_tx_desc *txdp, in vtx() argument
757 for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) in vtx()
758 vtx1(txdp, *pkt, flags); in vtx()
832 volatile struct fm10k_tx_desc *txdp; in fm10k_xmit_fixed_burst_vec() local
850 txdp = &txq->hw_ring[tx_id]; in fm10k_xmit_fixed_burst_vec()
860 vtx1(txdp, *tx_pkts, flags); in fm10k_xmit_fixed_burst_vec()
862 vtx1(txdp, *tx_pkts++, rs); in fm10k_xmit_fixed_burst_vec()
870 txdp = &(txq->hw_ring[tx_id]); in fm10k_xmit_fixed_burst_vec()
[all …]
|
| H A D | fm10k_rxtx.c | 438 volatile struct fm10k_tx_desc *txdp; in fm10k_dev_tx_descriptor_status() local
466 txdp = &txq->hw_ring[next_rs]; in fm10k_dev_tx_descriptor_status()
467 if (txdp->flags & FM10K_TXD_FLAG_DONE) in fm10k_dev_tx_descriptor_status()
|
| /dpdk/drivers/net/hns3/ |
| H A D | hns3_rxtx_vec_sve.c | 386 struct hns3_desc *txdp = &txq->tx_ring[txq->next_to_use]; in hns3_tx_fill_hw_ring_sve() local
423 svst1_scatter_u64offset_u64(pg, (uint64_t *)&txdp->addr, in hns3_tx_fill_hw_ring_sve()
426 svst1_scatter_u64offset_u64(pg, (uint64_t *)&txdp->tx.vlan_tag, in hns3_tx_fill_hw_ring_sve()
430 (uint64_t *)&txdp->tx.outer_vlan_tag, in hns3_tx_fill_hw_ring_sve()
434 (uint64_t *)&txdp->tx.paylen_fd_dop_ol4cs, in hns3_tx_fill_hw_ring_sve()
445 txdp += svcntd(); in hns3_tx_fill_hw_ring_sve()
|
| H A D | hns3_rxtx.c | 4051 txdp->tx.paylen_fd_dop_ol4cs = 0; in hns3_tx_setup_4bd()
4052 txdp->tx.type_cs_vlan_tso_len = 0; in hns3_tx_setup_4bd()
4053 txdp->tx.ol_type_vlan_len_msec = 0; in hns3_tx_setup_4bd()
4067 txdp->addr = rte_cpu_to_le_64(dma_addr); in hns3_tx_setup_1bd()
4069 txdp->tx.paylen_fd_dop_ol4cs = 0; in hns3_tx_setup_1bd()
4070 txdp->tx.type_cs_vlan_tso_len = 0; in hns3_tx_setup_1bd()
4071 txdp->tx.ol_type_vlan_len_msec = 0; in hns3_tx_setup_1bd()
4092 hns3_tx_setup_4bd(txdp + i, pkts + i); in hns3_tx_fill_hw_ring()
4103 hns3_tx_setup_1bd(txdp + mainpart + i, in hns3_tx_fill_hw_ring()
4668 volatile struct hns3_desc *txdp; in hns3_dev_tx_descriptor_status() local
[all …]
|
| /dpdk/drivers/net/ngbe/ |
| H A D | ngbe_rxtx.c | 110 tx4(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) in tx4() argument
116 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { in tx4()
121 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr); in tx4()
122 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | in tx4()
124 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); in tx4()
132 tx1(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) in tx1() argument
141 txdp->qw0 = cpu_to_le64(buf_dma_addr); in tx1()
142 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | in tx1()
144 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); in tx1()
174 tx4(txdp + i, pkts + i); in ngbe_tx_fill_hw_ring()
[all …]
|
| /dpdk/drivers/net/txgbe/ |
| H A D | txgbe_rxtx.c | 153 tx4(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts) in tx4() argument
159 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { in tx4()
164 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr); in tx4()
165 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS | in tx4()
167 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len)); in tx4()
175 tx1(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts) in tx1() argument
184 txdp->qw0 = cpu_to_le64(buf_dma_addr); in tx1()
185 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS | in tx1()
187 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len)); in tx1()
217 tx4(txdp + i, pkts + i); in txgbe_tx_fill_hw_ring()
[all …]
|