1 /******************************************************************************
2
3 Copyright (c) 2013-2018, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32 ******************************************************************************/
33 /*$FreeBSD$*/
34
35 #ifndef _IXL_H_
36 #define _IXL_H_
37
38 #include "opt_inet.h"
39 #include "opt_inet6.h"
40 #include "opt_rss.h"
41 #include "opt_ixl.h"
42
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/buf_ring.h>
46 #include <sys/mbuf.h>
47 #include <sys/protosw.h>
48 #include <sys/socket.h>
49 #include <sys/malloc.h>
50 #include <sys/kernel.h>
51 #include <sys/module.h>
52 #include <sys/sockio.h>
53 #include <sys/eventhandler.h>
54 #include <sys/syslog.h>
55 #include <sys/priv.h>
56
57 #include <net/if.h>
58 #include <net/if_var.h>
59 #include <net/if_arp.h>
60 #include <net/bpf.h>
61 #include <net/ethernet.h>
62 #include <net/if_dl.h>
63 #include <net/if_media.h>
64 #include <net/iflib.h>
65
66 #include <net/bpf.h>
67 #include <net/if_types.h>
68 #include <net/if_vlan_var.h>
69
70 #include <netinet/in_systm.h>
71 #include <netinet/in.h>
72 #include <netinet/if_ether.h>
73 #include <netinet/ip.h>
74 #include <netinet/ip6.h>
75 #include <netinet/tcp.h>
76 #include <netinet/tcp_lro.h>
77 #include <netinet/udp.h>
78 #include <netinet/sctp.h>
79
80 #include <machine/in_cksum.h>
81
82 #include <sys/bus.h>
83 #include <machine/bus.h>
84 #include <sys/rman.h>
85 #include <machine/resource.h>
86 #include <vm/vm.h>
87 #include <vm/pmap.h>
88 #include <machine/clock.h>
89 #include <dev/pci/pcivar.h>
90 #include <dev/pci/pcireg.h>
91 #include <sys/proc.h>
92 #include <sys/sysctl.h>
93 #include <sys/endian.h>
94 #include <sys/taskqueue.h>
95 #include <sys/pcpu.h>
96 #include <sys/smp.h>
97 #include <sys/sbuf.h>
98 #include <machine/smp.h>
99 #include <machine/stdarg.h>
100
101 #ifdef RSS
102 #include <net/rss_config.h>
103 #include <netinet/in_rss.h>
104 #endif
105
106 #include "ifdi_if.h"
107 #include "i40e_type.h"
108 #include "i40e_prototype.h"
109 #include "ixl_debug.h"
110
111 #define PVIDV(vendor, devid, name) \
112 PVID(vendor, devid, name " - " IXL_DRIVER_VERSION_STRING)
113
114 /* Tunables */
115
116 /*
117 * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the
118 * number of tx/rx descriptors allocated by the driver. Increasing this
119 * value allows the driver to queue more operations.
120 *
121 * Tx descriptors are always 16 bytes, but Rx descriptors can be 32 bytes.
122 * The driver currently always uses 32 byte Rx descriptors.
123 */
124 #define IXL_DEFAULT_RING 1024
125 #define IXL_MAX_RING 4096
126 #define IXL_MIN_RING 64
127 #define IXL_RING_INCREMENT 32
128
129 #define IXL_AQ_LEN 256
130 #define IXL_AQ_LEN_MAX 1024
131
132 /* Alignment for rings */
133 #define DBA_ALIGN 128
134
135 #define MAX_MULTICAST_ADDR 128
136
137 #define IXL_MSIX_BAR 3
138 #define IXL_ADM_LIMIT 2
139 #define IXL_TSO_SIZE ((255*1024)-1)
140 #define IXL_TX_BUF_SZ ((u32) 1514)
141 #define IXL_AQ_BUF_SZ ((u32) 4096)
142 #define IXL_RX_ITR 0
143 #define IXL_TX_ITR 1
144 #define IXL_ITR_NONE 3
145 #define IXL_QUEUE_EOL 0x7FF
146 #define IXL_MIN_FRAME 17
147 #define IXL_MAX_FRAME 9728
148 #define IXL_MAX_TX_SEGS 8
149 #define IXL_MAX_RX_SEGS 5
150 #define IXL_MAX_TSO_SEGS 128
151 #define IXL_SPARSE_CHAIN 7
152 #define IXL_MIN_TSO_MSS 64
153 #define IXL_MAX_TSO_MSS 9668
154 #define IXL_MAX_DMA_SEG_SIZE ((16 * 1024) - 1)
155
156 #define IXL_RSS_KEY_SIZE_REG 13
157 #define IXL_RSS_KEY_SIZE (IXL_RSS_KEY_SIZE_REG * 4)
158 #define IXL_RSS_VSI_LUT_SIZE 64 /* X722 -> VSI, X710 -> VF */
159 #define IXL_RSS_VSI_LUT_ENTRY_MASK 0x3F
160 #define IXL_RSS_VF_LUT_ENTRY_MASK 0xF
161
162 #define IXL_VF_MAX_BUFFER 0x3F80
163 #define IXL_VF_MAX_HDR_BUFFER 0x840
164 #define IXL_VF_MAX_FRAME 0x3FFF
165
166 /* ERJ: hardware can support ~2k (SW5+) filters between all functions */
167 #define IXL_MAX_FILTERS 256
168
169 #define IXL_NVM_VERSION_LO_SHIFT 0
170 #define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT)
171 #define IXL_NVM_VERSION_HI_SHIFT 12
172 #define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT)
173
174 /*
175 * Interrupt Moderation parameters
176 * Multiply ITR values by 2 for real ITR value
177 */
178 #define IXL_MAX_ITR 0x0FF0
179 #define IXL_ITR_100K 0x0005
180 #define IXL_ITR_20K 0x0019
181 #define IXL_ITR_8K 0x003E
182 #define IXL_ITR_4K 0x007A
183 #define IXL_ITR_1K 0x01F4
184 #define IXL_ITR_DYNAMIC 0x8000
185 #define IXL_LOW_LATENCY 0
186 #define IXL_AVE_LATENCY 1
187 #define IXL_BULK_LATENCY 2
188
189 /* MacVlan Flags */
190 #define IXL_FILTER_USED (u16)(1 << 0)
191 #define IXL_FILTER_VLAN (u16)(1 << 1)
192 #define IXL_FILTER_ADD (u16)(1 << 2)
193 #define IXL_FILTER_DEL (u16)(1 << 3)
194 #define IXL_FILTER_MC (u16)(1 << 4)
195
196 /* used in the vlan field of the filter when not a vlan */
197 #define IXL_VLAN_ANY -1
198
199 #define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP)
200 #define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6)
201 #define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO)
202
203 #define IXL_VF_RESET_TIMEOUT 100
204
205 #define IXL_VSI_DATA_PORT 0x01
206
207 #define IAVF_MAX_QUEUES 16
208 #define IXL_MAX_VSI_QUEUES (2 * (I40E_VSILAN_QTABLE_MAX_INDEX + 1))
209
210 #define IXL_RX_CTX_BASE_UNITS 128
211 #define IXL_TX_CTX_BASE_UNITS 128
212
213 #define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA
214
215 #define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20
216
217 #define IXL_GLGEN_VFLRSTAT_INDEX(glb_vf) ((glb_vf) / 32)
218 #define IXL_GLGEN_VFLRSTAT_MASK(glb_vf) (1 << ((glb_vf) % 32))
219
220 #define IXL_MAX_ITR_IDX 3
221
222 #define IXL_END_OF_INTR_LNKLST 0x7FF
223
224 #define IXL_DEFAULT_RSS_HENA_BASE (\
225 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \
226 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \
227 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \
228 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \
229 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \
230 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \
231 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \
232 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \
233 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \
234 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \
235 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD))
236
237 #define IXL_DEFAULT_RSS_HENA_XL710 IXL_DEFAULT_RSS_HENA_BASE
238
239 #define IXL_DEFAULT_RSS_HENA_X722 (\
240 IXL_DEFAULT_RSS_HENA_BASE | \
241 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \
242 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \
243 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \
244 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \
245 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \
246 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK))
247
248 #define IXL_CAPS \
249 (IFCAP_TSO4 | IFCAP_TSO6 | \
250 IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | \
251 IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | \
252 IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO | \
253 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \
254 IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU | IFCAP_LRO)
255
256 #define IXL_CSUM_TCP \
257 (CSUM_IP_TCP|CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP6_TCP)
258 #define IXL_CSUM_UDP \
259 (CSUM_IP_UDP|CSUM_IP6_UDP)
260 #define IXL_CSUM_SCTP \
261 (CSUM_IP_SCTP|CSUM_IP6_SCTP)
262 #define IXL_CSUM_IPV4 \
263 (CSUM_IP|CSUM_IP_TSO)
264
265 /* Pre-11 counter(9) compatibility */
266 #if __FreeBSD_version >= 1100036
267 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count)
268 #define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count)
269 #define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count)
270 #define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count)
271 #define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */
272 #define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count)
273 #define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count)
274 #define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count)
275 #define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count)
276 #define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count)
277 #define IXL_SET_OQDROPS(vsi, count) (vsi)->oqdrops = (count)
278 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count)
279 #else
280 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ifp->if_ipackets = (count)
281 #define IXL_SET_IERRORS(vsi, count) (vsi)->ifp->if_ierrors = (count)
282 #define IXL_SET_OPACKETS(vsi, count) (vsi)->ifp->if_opackets = (count)
283 #define IXL_SET_OERRORS(vsi, count) (vsi)->ifp->if_oerrors = (count)
284 #define IXL_SET_COLLISIONS(vsi, count) (vsi)->ifp->if_collisions = (count)
285 #define IXL_SET_IBYTES(vsi, count) (vsi)->ifp->if_ibytes = (count)
286 #define IXL_SET_OBYTES(vsi, count) (vsi)->ifp->if_obytes = (count)
287 #define IXL_SET_IMCASTS(vsi, count) (vsi)->ifp->if_imcasts = (count)
288 #define IXL_SET_OMCASTS(vsi, count) (vsi)->ifp->if_omcasts = (count)
289 #define IXL_SET_IQDROPS(vsi, count) (vsi)->ifp->if_iqdrops = (count)
290 #define IXL_SET_OQDROPS(vsi, odrops) (vsi)->ifp->if_snd.ifq_drops = (odrops)
291 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count)
292 #endif
293
294 /* For stats sysctl naming */
295 #define QUEUE_NAME_LEN 32
296
297 #define IXL_DEV_ERR(_dev, _format, ...) \
298 device_printf(_dev, "%s: " _format " (%s:%d)\n", __func__, ##__VA_ARGS__, __FILE__, __LINE__)
299
300 /*
301 *****************************************************************************
302 * vendor_info_array
303 *
304 * This array contains the list of Subvendor/Subdevice IDs on which the driver
305 * should load.
306 *
307 *****************************************************************************
308 */
309 typedef struct _ixl_vendor_info_t {
310 unsigned int vendor_id;
311 unsigned int device_id;
312 unsigned int subvendor_id;
313 unsigned int subdevice_id;
314 unsigned int index;
315 } ixl_vendor_info_t;
316
317 /*
318 ** This struct has multiple uses, multicast
319 ** addresses, vlans, and mac filters all use it.
320 */
321 struct ixl_mac_filter {
322 SLIST_ENTRY(ixl_mac_filter) next;
323 u8 macaddr[ETHER_ADDR_LEN];
324 s16 vlan;
325 u16 flags;
326 };
327
328 /*
329 * The Transmit ring control struct
330 */
331 struct tx_ring {
332 struct ixl_tx_queue *que;
333 u32 tail;
334 struct i40e_tx_desc *tx_base;
335 u64 tx_paddr;
336 u32 latency;
337 u32 packets;
338 u32 me;
339 /*
340 * For reporting completed packet status
341 * in descriptor writeback mode
342 */
343 qidx_t *tx_rsq;
344 qidx_t tx_rs_cidx;
345 qidx_t tx_rs_pidx;
346 qidx_t tx_cidx_processed;
347
348 /* Used for Dynamic ITR calculation */
349 u32 itr;
350 u32 bytes;
351
352 /* Soft Stats */
353 u64 tx_bytes;
354 u64 tx_packets;
355 u64 mss_too_small;
356 };
357
358
359 /*
360 * The Receive ring control struct
361 */
362 struct rx_ring {
363 struct ixl_rx_queue *que;
364 union i40e_rx_desc *rx_base;
365 uint64_t rx_paddr;
366 bool discard;
367 u32 itr;
368 u32 latency;
369 u32 mbuf_sz;
370 u32 tail;
371 u32 me;
372
373 /* Used for Dynamic ITR calculation */
374 u32 packets;
375 u32 bytes;
376
377 /* Soft stats */
378 u64 rx_packets;
379 u64 rx_bytes;
380 u64 desc_errs;
381 };
382
383 /*
384 ** Driver queue structs
385 */
386 struct ixl_tx_queue {
387 struct ixl_vsi *vsi;
388 struct tx_ring txr;
389 struct if_irq que_irq;
390 u32 msix;
391 /* Stats */
392 u64 irqs;
393 u64 tso;
394 };
395
396 struct ixl_rx_queue {
397 struct ixl_vsi *vsi;
398 struct rx_ring rxr;
399 struct if_irq que_irq;
400 u32 msix; /* This queue's MSIX vector */
401 /* Stats */
402 u64 irqs;
403 };
404
405 /*
406 ** Virtual Station Interface
407 */
408 SLIST_HEAD(ixl_ftl_head, ixl_mac_filter);
409 struct ixl_vsi {
410 if_ctx_t ctx;
411 if_softc_ctx_t shared;
412 struct ifnet *ifp;
413 device_t dev;
414 struct i40e_hw *hw;
415 struct ifmedia *media;
416
417 int num_rx_queues;
418 int num_tx_queues;
419
420 void *back;
421 enum i40e_vsi_type type;
422 int id;
423 u32 rx_itr_setting;
424 u32 tx_itr_setting;
425 bool enable_head_writeback;
426
427 u16 vsi_num;
428 bool link_active;
429 u16 seid;
430 u16 uplink_seid;
431 u16 downlink_seid;
432
433 struct ixl_tx_queue *tx_queues; /* TX queue array */
434 struct ixl_rx_queue *rx_queues; /* RX queue array */
435 struct if_irq irq;
436 u32 link_speed;
437
438 /* MAC/VLAN Filter list */
439 struct ixl_ftl_head ftl;
440 u16 num_macs;
441
442 /* Contains readylist & stat counter id */
443 struct i40e_aqc_vsi_properties_data info;
444
445 u16 num_vlans;
446
447 /* Per-VSI stats from hardware */
448 struct i40e_eth_stats eth_stats;
449 struct i40e_eth_stats eth_stats_offsets;
450 bool stat_offsets_loaded;
451 /* VSI stat counters */
452 u64 ipackets;
453 u64 ierrors;
454 u64 opackets;
455 u64 oerrors;
456 u64 ibytes;
457 u64 obytes;
458 u64 imcasts;
459 u64 omcasts;
460 u64 iqdrops;
461 u64 oqdrops;
462 u64 noproto;
463
464 /* Driver statistics */
465 u64 hw_filters_del;
466 u64 hw_filters_add;
467
468 /* Misc. */
469 u64 flags;
470 /* Stats sysctls for this VSI */
471 struct sysctl_oid *vsi_node;
472 };
473
474 /*
475 ** Creates new filter with given MAC address and VLAN ID
476 */
477 static inline struct ixl_mac_filter *
ixl_new_filter(struct ixl_vsi * vsi,const u8 * macaddr,s16 vlan)478 ixl_new_filter(struct ixl_vsi *vsi, const u8 *macaddr, s16 vlan)
479 {
480 struct ixl_mac_filter *f;
481
482 /* create a new empty filter */
483 f = malloc(sizeof(struct ixl_mac_filter),
484 M_DEVBUF, M_NOWAIT | M_ZERO);
485 if (f) {
486 SLIST_INSERT_HEAD(&vsi->ftl, f, next);
487 bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
488 f->vlan = vlan;
489 f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
490 }
491
492 return (f);
493 }
494
495 /*
496 ** Compare two ethernet addresses
497 */
498 static inline bool
cmp_etheraddr(const u8 * ea1,const u8 * ea2)499 cmp_etheraddr(const u8 *ea1, const u8 *ea2)
500 {
501 return (bcmp(ea1, ea2, 6) == 0);
502 }
503
504 /*
505 * Return next largest power of 2, unsigned
506 *
507 * Public domain, from Bit Twiddling Hacks
508 */
509 static inline u32
next_power_of_two(u32 n)510 next_power_of_two(u32 n)
511 {
512 n--;
513 n |= n >> 1;
514 n |= n >> 2;
515 n |= n >> 4;
516 n |= n >> 8;
517 n |= n >> 16;
518 n++;
519
520 /* Next power of two > 0 is 1 */
521 n += (n == 0);
522
523 return (n);
524 }
525
526 /*
527 * Info for stats sysctls
528 */
529 struct ixl_sysctl_info {
530 u64 *stat;
531 char *name;
532 char *description;
533 };
534
535 extern const uint8_t ixl_bcast_addr[ETHER_ADDR_LEN];
536
537 /* Common function prototypes between PF/VF driver */
538 void ixl_debug_core(device_t dev, u32 enabled_mask, u32 mask, char *fmt, ...);
539 void ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que);
540 void ixl_get_default_rss_key(u32 *);
541 const char * i40e_vc_stat_str(struct i40e_hw *hw,
542 enum virtchnl_status_code stat_err);
543 void ixl_init_tx_rsqs(struct ixl_vsi *vsi);
544 void ixl_init_tx_cidx(struct ixl_vsi *vsi);
545 u64 ixl_max_vc_speed_to_value(u8 link_speeds);
546 void ixl_add_vsi_sysctls(device_t dev, struct ixl_vsi *vsi,
547 struct sysctl_ctx_list *ctx, const char *sysctl_name);
548 void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
549 struct sysctl_oid_list *child,
550 struct i40e_eth_stats *eth_stats);
551 void ixl_add_queues_sysctls(device_t dev, struct ixl_vsi *vsi);
552 #endif /* _IXL_H_ */
553