1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2016-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 #ifndef _SFC_DP_TX_H 11 #define _SFC_DP_TX_H 12 13 #include <ethdev_driver.h> 14 15 #include "sfc_dp.h" 16 #include "sfc_debug.h" 17 #include "sfc_tso.h" 18 19 #ifdef __cplusplus 20 extern "C" { 21 #endif 22 23 /** 24 * Generic transmit queue information used on data path. 25 * It must be kept as small as it is possible since it is built into 26 * the structure used on datapath. 27 */ 28 struct sfc_dp_txq { 29 struct sfc_dp_queue dpq; 30 }; 31 32 /** Datapath transmit queue descriptor number limitations */ 33 struct sfc_dp_tx_hw_limits { 34 unsigned int txq_max_entries; 35 unsigned int txq_min_entries; 36 }; 37 38 /** 39 * Datapath transmit queue creation information. 40 * 41 * The structure is used just to pass information from control path to 42 * datapath. It could be just function arguments, but it would be hardly 43 * readable. 44 */ 45 struct sfc_dp_tx_qcreate_info { 46 /** Maximum number of pushed Tx descriptors */ 47 unsigned int max_fill_level; 48 /** Minimum number of unused Tx descriptors to do reap */ 49 unsigned int free_thresh; 50 /** Offloads enabled on the transmit queue */ 51 uint64_t offloads; 52 /** Tx queue size */ 53 unsigned int txq_entries; 54 /** Maximum size of data in the DMA descriptor */ 55 uint16_t dma_desc_size_max; 56 /** DMA-mapped Tx descriptors ring */ 57 void *txq_hw_ring; 58 /** Associated event queue size */ 59 unsigned int evq_entries; 60 /** Hardware event ring */ 61 void *evq_hw_ring; 62 /** The queue index in hardware (required to push right doorbell) */ 63 unsigned int hw_index; 64 /** Virtual address of the memory-mapped BAR to push Tx doorbell */ 65 volatile void *mem_bar; 66 /** VI window size shift */ 67 unsigned int vi_window_shift; 68 /** 69 * Maximum number of bytes into the packet the TCP header can start for 70 * the hardware to apply TSO packet edits. 71 */ 72 uint16_t tso_tcp_header_offset_limit; 73 /** Maximum number of header DMA descriptors per TSOv3 transaction */ 74 uint16_t tso_max_nb_header_descs; 75 /** Maximum header length acceptable by TSOv3 transaction */ 76 uint16_t tso_max_header_len; 77 /** Maximum number of payload DMA descriptors per TSOv3 transaction */ 78 uint16_t tso_max_nb_payload_descs; 79 /** Maximum payload length per TSOv3 transaction */ 80 uint32_t tso_max_payload_len; 81 /** Maximum number of frames to be generated per TSOv3 transaction */ 82 uint32_t tso_max_nb_outgoing_frames; 83 }; 84 85 /** 86 * Get Tx datapath specific device info. 87 * 88 * @param dev_info Device info to be adjusted 89 */ 90 typedef void (sfc_dp_tx_get_dev_info_t)(struct rte_eth_dev_info *dev_info); 91 92 /** 93 * Get size of transmit and event queue rings by the number of Tx 94 * descriptors. 95 * 96 * @param nb_tx_desc Number of Tx descriptors 97 * @param txq_entries Location for number of Tx ring entries 98 * @param evq_entries Location for number of event ring entries 99 * @param txq_max_fill_level Location for maximum Tx ring fill level 100 * 101 * @return 0 or positive errno. 102 */ 103 typedef int (sfc_dp_tx_qsize_up_rings_t)(uint16_t nb_tx_desc, 104 struct sfc_dp_tx_hw_limits *limits, 105 unsigned int *txq_entries, 106 unsigned int *evq_entries, 107 unsigned int *txq_max_fill_level); 108 109 /** 110 * Allocate and initialize datapath transmit queue. 111 * 112 * @param port_id The port identifier 113 * @param queue_id The queue identifier 114 * @param pci_addr PCI function address 115 * @param socket_id Socket identifier to allocate memory 116 * @param info Tx queue details wrapped in structure 117 * @param dp_txqp Location for generic datapath transmit queue pointer 118 * 119 * @return 0 or positive errno. 120 */ 121 typedef int (sfc_dp_tx_qcreate_t)(uint16_t port_id, uint16_t queue_id, 122 const struct rte_pci_addr *pci_addr, 123 int socket_id, 124 const struct sfc_dp_tx_qcreate_info *info, 125 struct sfc_dp_txq **dp_txqp); 126 127 /** 128 * Free resources allocated for datapath transmit queue. 129 */ 130 typedef void (sfc_dp_tx_qdestroy_t)(struct sfc_dp_txq *dp_txq); 131 132 /** 133 * Transmit queue start callback. 134 * 135 * It handovers EvQ to the datapath. 136 */ 137 typedef int (sfc_dp_tx_qstart_t)(struct sfc_dp_txq *dp_txq, 138 unsigned int evq_read_ptr, 139 unsigned int txq_desc_index); 140 141 /** 142 * Transmit queue stop function called before the queue flush. 143 * 144 * It returns EvQ to the control path. 145 */ 146 typedef void (sfc_dp_tx_qstop_t)(struct sfc_dp_txq *dp_txq, 147 unsigned int *evq_read_ptr); 148 149 /** 150 * Transmit event handler used during queue flush only. 151 */ 152 typedef bool (sfc_dp_tx_qtx_ev_t)(struct sfc_dp_txq *dp_txq, unsigned int id); 153 154 /** 155 * Transmit queue function called after the queue flush. 156 */ 157 typedef void (sfc_dp_tx_qreap_t)(struct sfc_dp_txq *dp_txq); 158 159 /** 160 * Check Tx descriptor status 161 */ 162 typedef int (sfc_dp_tx_qdesc_status_t)(struct sfc_dp_txq *dp_txq, 163 uint16_t offset); 164 165 /** Transmit datapath definition */ 166 struct sfc_dp_tx { 167 struct sfc_dp dp; 168 169 unsigned int features; 170 #define SFC_DP_TX_FEAT_MULTI_PROCESS 0x1 171 #define SFC_DP_TX_FEAT_STATS 0x2 172 /** 173 * Tx offload capabilities supported by the datapath on device 174 * level only if HW/FW supports it. 175 */ 176 uint64_t dev_offload_capa; 177 /** 178 * Tx offload capabilities supported by the datapath per-queue 179 * if HW/FW supports it. 180 */ 181 uint64_t queue_offload_capa; 182 sfc_dp_tx_get_dev_info_t *get_dev_info; 183 sfc_dp_tx_qsize_up_rings_t *qsize_up_rings; 184 sfc_dp_tx_qcreate_t *qcreate; 185 sfc_dp_tx_qdestroy_t *qdestroy; 186 sfc_dp_tx_qstart_t *qstart; 187 sfc_dp_tx_qstop_t *qstop; 188 sfc_dp_tx_qtx_ev_t *qtx_ev; 189 sfc_dp_tx_qreap_t *qreap; 190 sfc_dp_tx_qdesc_status_t *qdesc_status; 191 eth_tx_prep_t pkt_prepare; 192 eth_tx_burst_t pkt_burst; 193 }; 194 195 static inline struct sfc_dp_tx * 196 sfc_dp_find_tx_by_name(struct sfc_dp_list *head, const char *name) 197 { 198 struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_TX, name); 199 200 return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp); 201 } 202 203 static inline struct sfc_dp_tx * 204 sfc_dp_find_tx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps) 205 { 206 struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_TX, avail_caps); 207 208 return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp); 209 } 210 211 /** Get Tx datapath ops by the datapath TxQ handle */ 212 const struct sfc_dp_tx *sfc_dp_tx_by_dp_txq(const struct sfc_dp_txq *dp_txq); 213 214 static inline uint64_t 215 sfc_dp_tx_offload_capa(const struct sfc_dp_tx *dp_tx) 216 { 217 return dp_tx->dev_offload_capa | dp_tx->queue_offload_capa; 218 } 219 220 static inline unsigned int 221 sfc_dp_tx_pkt_extra_hdr_segs(struct rte_mbuf **m_seg, 222 unsigned int *header_len_remaining) 223 { 224 unsigned int nb_extra_header_segs = 0; 225 226 while (rte_pktmbuf_data_len(*m_seg) < *header_len_remaining) { 227 *header_len_remaining -= rte_pktmbuf_data_len(*m_seg); 228 *m_seg = (*m_seg)->next; 229 ++nb_extra_header_segs; 230 } 231 232 return nb_extra_header_segs; 233 } 234 235 static inline int 236 sfc_dp_tx_prepare_pkt(struct rte_mbuf *m, 237 unsigned int max_nb_header_segs, 238 unsigned int tso_bounce_buffer_len, 239 uint32_t tso_tcp_header_offset_limit, 240 unsigned int max_fill_level, 241 unsigned int nb_tso_descs, 242 unsigned int nb_vlan_descs) 243 { 244 unsigned int descs_required = m->nb_segs; 245 unsigned int tcph_off = ((m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) ? 246 m->outer_l2_len + m->outer_l3_len : 0) + 247 m->l2_len + m->l3_len; 248 unsigned int header_len = tcph_off + m->l4_len; 249 unsigned int header_len_remaining = header_len; 250 unsigned int nb_header_segs = 1; 251 struct rte_mbuf *m_seg = m; 252 253 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG 254 int ret; 255 256 ret = rte_validate_tx_offload(m); 257 if (ret != 0) { 258 /* 259 * Negative error code is returned by rte_validate_tx_offload(), 260 * but positive are used inside net/sfc PMD. 261 */ 262 SFC_ASSERT(ret < 0); 263 return -ret; 264 } 265 #endif 266 267 if (max_nb_header_segs != 0) { 268 /* There is a limit on the number of header segments. */ 269 270 nb_header_segs += 271 sfc_dp_tx_pkt_extra_hdr_segs(&m_seg, 272 &header_len_remaining); 273 274 if (unlikely(nb_header_segs > max_nb_header_segs)) { 275 /* 276 * The number of header segments is too large. 277 * 278 * If TSO is requested and if the datapath supports 279 * linearisation of TSO headers, allow the packet 280 * to proceed with additional checks below. 281 * Otherwise, throw an error. 282 */ 283 if ((m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) == 0 || 284 tso_bounce_buffer_len == 0) 285 return EINVAL; 286 } 287 } 288 289 if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 290 switch (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 291 case 0: 292 break; 293 case RTE_MBUF_F_TX_TUNNEL_VXLAN: 294 /* FALLTHROUGH */ 295 case RTE_MBUF_F_TX_TUNNEL_GENEVE: 296 if (!(m->ol_flags & 297 (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IPV6))) 298 return EINVAL; 299 } 300 301 if (unlikely(tcph_off > tso_tcp_header_offset_limit)) 302 return EINVAL; 303 304 descs_required += nb_tso_descs; 305 306 /* 307 * If headers segments are already counted above, here 308 * nothing is done since remaining length is smaller 309 * then current segment size. 310 */ 311 nb_header_segs += 312 sfc_dp_tx_pkt_extra_hdr_segs(&m_seg, 313 &header_len_remaining); 314 315 /* 316 * Extra descriptor which is required when (a part of) payload 317 * shares the same segment with (a part of) the header. 318 */ 319 if (rte_pktmbuf_data_len(m_seg) > header_len_remaining) 320 descs_required++; 321 322 if (tso_bounce_buffer_len != 0) { 323 if (nb_header_segs > 1 && 324 unlikely(header_len > tso_bounce_buffer_len)) { 325 /* 326 * Header linearization is required and 327 * the header is too big to be linearized 328 */ 329 return EINVAL; 330 } 331 } 332 } 333 334 /* 335 * The number of VLAN descriptors is added regardless of requested 336 * VLAN offload since VLAN is sticky and sending packet without VLAN 337 * insertion may require VLAN descriptor to reset the sticky to 0. 338 */ 339 descs_required += nb_vlan_descs; 340 341 /* 342 * Max fill level must be sufficient to hold all required descriptors 343 * to send the packet entirely. 344 */ 345 if (descs_required > max_fill_level) 346 return ENOBUFS; 347 348 return 0; 349 } 350 351 extern struct sfc_dp_tx sfc_efx_tx; 352 extern struct sfc_dp_tx sfc_ef10_tx; 353 extern struct sfc_dp_tx sfc_ef10_simple_tx; 354 extern struct sfc_dp_tx sfc_ef100_tx; 355 356 #ifdef __cplusplus 357 } 358 #endif 359 #endif /* _SFC_DP_TX_H */ 360