| /f-stack/dpdk/drivers/net/thunderx/ |
| H A D | nicvf_rxtx.h | 60 nicvf_mbuff_init_update(struct rte_mbuf *pkt, const uint64_t mbuf_init, in nicvf_mbuff_init_update() argument
63 union mbuf_initializer init = {.value = mbuf_init}; in nicvf_mbuff_init_update()
73 nicvf_mbuff_init_mseg_update(struct rte_mbuf *pkt, const uint64_t mbuf_init, in nicvf_mbuff_init_mseg_update() argument
76 union mbuf_initializer init = {.value = mbuf_init}; in nicvf_mbuff_init_mseg_update()
|
| H A D | nicvf_rxtx.c | 432 const uint64_t mbuf_init = rxq->mbuf_initializer.value; in nicvf_recv_pkts() local
466 nicvf_mbuff_init_update(pkt, mbuf_init, cqe_rx_w1.align_pad); in nicvf_recv_pkts()
523 uint64_t mbuf_init, const uint32_t flag) in nicvf_process_cq_mseg_entry() argument
544 pkt, mbuf_init, cqe_rx_w1.align_pad, nb_segs); in nicvf_process_cq_mseg_entry()
567 nicvf_mbuff_init_update(seg, mbuf_init, 0); in nicvf_process_cq_mseg_entry()
588 const uint64_t mbuf_init = rxq->mbuf_initializer.value; in nicvf_recv_pkts_multiseg() local
599 rx_pkts + i, rbptr_offset, mbuf_init, flag); in nicvf_recv_pkts_multiseg()
|
| /f-stack/dpdk/drivers/net/iavf/ |
| H A D | iavf_rxtx_vec_avx2.c | 496 rearm6 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx2()
499 rearm4 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx2()
503 rearm0 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx2()
525 rearm7 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx2()
528 rearm5 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx2()
532 rearm1 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx2()
1105 rearm6 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx2_flex_rxd()
1108 rearm4 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx2_flex_rxd()
1112 rearm0 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx2_flex_rxd()
1134 rearm7 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx2_flex_rxd()
[all …]
|
| H A D | iavf_rxtx_vec_avx512.c | 496 rearm6 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx512()
499 rearm4 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx512()
503 rearm0 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx512()
525 rearm7 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx512()
528 rearm5 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx512()
532 rearm1 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx512()
1120 rearm6 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx512_flex_rxd()
1123 rearm4 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx512_flex_rxd()
1127 rearm0 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx512_flex_rxd()
1149 rearm7 = _mm256_blend_epi32(mbuf_init, in _iavf_recv_raw_pkts_vec_avx512_flex_rxd()
[all …]
|
| H A D | iavf_rxtx_vec_sse.c | 98 const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); in desc_to_olflags_v() local
176 rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vlan0, 8), 0x10); in desc_to_olflags_v()
177 rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vlan0, 4), 0x10); in desc_to_olflags_v()
178 rearm2 = _mm_blend_epi16(mbuf_init, vlan0, 0x10); in desc_to_olflags_v()
179 rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(vlan0, 4), 0x10); in desc_to_olflags_v()
215 const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); in flex_desc_to_olflags_v() local
328 rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(flags, 8), 0x10); in flex_desc_to_olflags_v()
329 rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(flags, 4), 0x10); in flex_desc_to_olflags_v()
330 rearm2 = _mm_blend_epi16(mbuf_init, flags, 0x10); in flex_desc_to_olflags_v()
331 rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(flags, 4), 0x10); in flex_desc_to_olflags_v()
|
| /f-stack/dpdk/drivers/net/bnxt/ |
| H A D | bnxt_rxtx_vec_sse.c | 57 __m128i mbuf_init, struct rte_mbuf **mbuf) in descs_to_mbufs() argument
113 _mm_or_si128(mbuf_init, _mm_set_epi64x(ol_flags, 0))); in descs_to_mbufs()
117 _mm_or_si128(mbuf_init, _mm_set_epi64x(ol_flags, 0))); in descs_to_mbufs()
121 _mm_or_si128(mbuf_init, _mm_set_epi64x(ol_flags, 0))); in descs_to_mbufs()
125 _mm_or_si128(mbuf_init, _mm_set_epi64x(ol_flags, 0))); in descs_to_mbufs()
146 const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); in bnxt_recv_pkts_vec() local
271 descs_to_mbufs(rxcmp, rxcmp1, mbuf_init, &rx_pkts[nb_rx_pkts]); in bnxt_recv_pkts_vec()
|
| /f-stack/dpdk/drivers/net/ixgbe/ |
| H A D | ixgbe_rxtx_vec_sse.c | 134 desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, in desc_to_olflags_v() argument
227 rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 8), 0x10); in desc_to_olflags_v()
228 rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 6), 0x10); in desc_to_olflags_v()
229 rearm2 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 4), 0x10); in desc_to_olflags_v()
230 rearm3 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 2), 0x10); in desc_to_olflags_v()
342 __m128i mbuf_init; in _recv_raw_pkts_vec() local
398 mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); in _recv_raw_pkts_vec()
480 desc_to_olflags_v(descs, mbuf_init, vlan_flags, &rx_pkts[pos]); in _recv_raw_pkts_vec()
|
| /f-stack/dpdk/drivers/net/ice/ |
| H A D | ice_rxtx_vec_avx2.c | 161 const __m256i mbuf_init = _mm256_set_epi64x(0, 0, in _ice_recv_raw_pkts_vec_avx2() local
637 rearm6 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx2()
640 rearm4 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx2()
643 rearm2 = _mm256_blend_epi32(mbuf_init, mbuf_flags, 0x04); in _ice_recv_raw_pkts_vec_avx2()
644 rearm0 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx2()
666 rearm7 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx2()
669 rearm5 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx2()
672 rearm3 = _mm256_blend_epi32(mbuf_init, odd_flags, 0x04); in _ice_recv_raw_pkts_vec_avx2()
673 rearm1 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx2()
|
| H A D | ice_rxtx_vec_avx512.c | 156 const __m256i mbuf_init = _mm256_set_epi64x(0, 0, in _ice_recv_raw_pkts_vec_avx512() local
621 rearm6 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx512()
624 rearm4 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx512()
627 rearm2 = _mm256_blend_epi32(mbuf_init, mbuf_flags, 0x04); in _ice_recv_raw_pkts_vec_avx512()
628 rearm0 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx512()
652 rearm7 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx512()
655 rearm5 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx512()
658 rearm3 = _mm256_blend_epi32(mbuf_init, odd_flags, 0x04); in _ice_recv_raw_pkts_vec_avx512()
659 rearm1 = _mm256_blend_epi32(mbuf_init, in _ice_recv_raw_pkts_vec_avx512()
|
| H A D | ice_rxtx_vec_sse.c | 107 const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); in ice_rx_desc_to_olflags_v() local
220 rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(flags, 8), 0x10); in ice_rx_desc_to_olflags_v()
221 rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(flags, 4), 0x10); in ice_rx_desc_to_olflags_v()
222 rearm2 = _mm_blend_epi16(mbuf_init, flags, 0x10); in ice_rx_desc_to_olflags_v()
223 rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(flags, 4), 0x10); in ice_rx_desc_to_olflags_v()
|
| /f-stack/dpdk/drivers/net/i40e/ |
| H A D | i40e_rxtx_vec_avx2.c | 230 const __m256i mbuf_init = _mm256_set_epi64x(0, 0, in _recv_raw_pkts_vec_avx2() local
640 rearm6 = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 8), 0x04); in _recv_raw_pkts_vec_avx2()
641 rearm4 = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 4), 0x04); in _recv_raw_pkts_vec_avx2()
642 rearm2 = _mm256_blend_epi32(mbuf_init, mbuf_flags, 0x04); in _recv_raw_pkts_vec_avx2()
643 rearm0 = _mm256_blend_epi32(mbuf_init, _mm256_srli_si256(mbuf_flags, 4), 0x04); in _recv_raw_pkts_vec_avx2()
658 rearm7 = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(odd_flags, 8), 0x04); in _recv_raw_pkts_vec_avx2()
659 rearm5 = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(odd_flags, 4), 0x04); in _recv_raw_pkts_vec_avx2()
660 rearm3 = _mm256_blend_epi32(mbuf_init, odd_flags, 0x04); in _recv_raw_pkts_vec_avx2()
661 rearm1 = _mm256_blend_epi32(mbuf_init, _mm256_srli_si256(odd_flags, 4), 0x04); in _recv_raw_pkts_vec_avx2()
|
| H A D | i40e_rxtx_vec_neon.c | 85 const uint64x2_t mbuf_init = {rxq->mbuf_initializer, 0}; in desc_to_olflags_v() local
160 rearm0 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 0), mbuf_init, 1); in desc_to_olflags_v()
161 rearm1 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 1), mbuf_init, 1); in desc_to_olflags_v()
162 rearm2 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 2), mbuf_init, 1); in desc_to_olflags_v()
163 rearm3 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 3), mbuf_init, 1); in desc_to_olflags_v()
|
| H A D | i40e_rxtx_vec_sse.c | 218 const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); in desc_to_olflags_v() local
311 rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vlan0, 8), 0x10); in desc_to_olflags_v()
312 rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vlan0, 4), 0x10); in desc_to_olflags_v()
313 rearm2 = _mm_blend_epi16(mbuf_init, vlan0, 0x10); in desc_to_olflags_v()
314 rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(vlan0, 4), 0x10); in desc_to_olflags_v()
|
| /f-stack/dpdk/drivers/net/enic/ |
| H A D | enic_rxtx_vec_avx2.c | 381 const __m256i mbuf_init = _mm256_set_epi64x(0, enic->mbuf_initializer, in enic_noscatter_vec_recv_pkts() local
687 __m256i mbuf_init4_5 = _mm256_blend_epi32(mbuf_init, in enic_noscatter_vec_recv_pkts()
689 __m256i mbuf_init2_3 = _mm256_blend_epi32(mbuf_init, in enic_noscatter_vec_recv_pkts()
691 __m256i mbuf_init0_1 = _mm256_blend_epi32(mbuf_init, in enic_noscatter_vec_recv_pkts()
693 __m256i mbuf_init6_7 = _mm256_blend_epi32(mbuf_init, in enic_noscatter_vec_recv_pkts()
|
| /f-stack/dpdk/drivers/net/hns3/ |
| H A D | hns3_rxtx_vec_sve.c | 124 svuint64_t vld_clz, mbp1st, mbp2st, mbuf_init; in hns3_recv_burst_vec_sve() local
183 mbuf_init = svdup_n_u64(rxq->mbuf_initializer); in hns3_recv_burst_vec_sve()
197 offsetof(struct rte_mbuf, rearm_data), mbuf_init); in hns3_recv_burst_vec_sve()
199 offsetof(struct rte_mbuf, rearm_data), mbuf_init); in hns3_recv_burst_vec_sve()
|
| /f-stack/dpdk/drivers/net/ark/ |
| H A D | ark_ethdev_rx.c | 470 struct rte_mbuf *mbuf_init = in eth_ark_rx_seed_mbufs() local
473 memset(mbuf_init->buf_addr, -1, 512); in eth_ark_rx_seed_mbufs()
474 *((uint32_t *)mbuf_init->buf_addr) = in eth_ark_rx_seed_mbufs()
476 *(uint16_t *)RTE_PTR_ADD(mbuf_init->buf_addr, 4) = in eth_ark_rx_seed_mbufs()
|
| /f-stack/dpdk/drivers/net/mlx5/ |
| H A D | mlx5_rxtx_vec_neon.h | 415 const uint64x2_t mbuf_init = vld1q_u64 in rxq_cq_to_ptype_oflags_v() local
479 vreinterpretq_u32_u64(mbuf_init), 2)); in rxq_cq_to_ptype_oflags_v()
482 vreinterpretq_u32_u64(mbuf_init), 2)); in rxq_cq_to_ptype_oflags_v()
485 vreinterpretq_u32_u64(mbuf_init), 2)); in rxq_cq_to_ptype_oflags_v()
488 vreinterpretq_u32_u64(mbuf_init), 2)); in rxq_cq_to_ptype_oflags_v()
|
| H A D | mlx5_rxtx_vec_sse.h | 404 const __m128i mbuf_init = in rxq_cq_to_ptype_oflags_v() local
484 rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 8), 0x30); in rxq_cq_to_ptype_oflags_v()
485 rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 4), 0x30); in rxq_cq_to_ptype_oflags_v()
486 rearm2 = _mm_blend_epi16(mbuf_init, ol_flags, 0x30); in rxq_cq_to_ptype_oflags_v()
487 rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(ol_flags, 4), 0x30); in rxq_cq_to_ptype_oflags_v()
|
| H A D | mlx5_rxtx_vec_altivec.h | 569 const vector unsigned char mbuf_init = in rxq_cq_to_ptype_oflags_v() local
726 vec_sel((vector unsigned short)mbuf_init, in rxq_cq_to_ptype_oflags_v()
731 vec_sel((vector unsigned short)mbuf_init, in rxq_cq_to_ptype_oflags_v()
736 vec_sel((vector unsigned short)mbuf_init, in rxq_cq_to_ptype_oflags_v()
739 vec_sel((vector unsigned short)mbuf_init, in rxq_cq_to_ptype_oflags_v()
|
| H A D | mlx5_rxq.c | 241 struct rte_mbuf *mbuf_init = &rxq->fake_mbuf; in rxq_alloc_elts_sprq() local
248 mbuf_init->data_off = RTE_PKTMBUF_HEADROOM; in rxq_alloc_elts_sprq()
249 rte_mbuf_refcnt_set(mbuf_init, 1); in rxq_alloc_elts_sprq()
250 mbuf_init->nb_segs = 1; in rxq_alloc_elts_sprq()
251 mbuf_init->port = rxq->port_id; in rxq_alloc_elts_sprq()
253 mbuf_init->ol_flags = EXT_ATTACHED_MBUF; in rxq_alloc_elts_sprq()
260 *(rte_xmm_t *)&mbuf_init->rearm_data; in rxq_alloc_elts_sprq()
|
| /f-stack/dpdk/drivers/event/octeontx2/ |
| H A D | otx2_evdev.h | 232 static const union mbuf_initializer mbuf_init = { variable
247 uint64_t val = mbuf_init.value | (uint64_t)port_id << 48; in otx2_wqe_to_mbuf()
|
| /f-stack/dpdk/drivers/net/octeontx2/ |
| H A D | otx2_rx.h | 109 union mbuf_initializer mbuf_init = { .value = oldval }; in nix_clear_data_off() local
111 mbuf_init.fields.data_off = 0; in nix_clear_data_off()
112 return mbuf_init.value; in nix_clear_data_off()
|
| H A D | otx2_rx.c | 49 const uint64_t mbuf_init = rxq->mbuf_initializer; in nix_recv_pkts() local
70 otx2_nix_cqe_to_mbuf(cq, cq->tag, mbuf, lookup_mem, mbuf_init, in nix_recv_pkts()
|
| /f-stack/freebsd/kern/ |
| H A D | kern_mbuf.c | 326 mbuf_init(void *dummy) in mbuf_init() function
386 SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
|