1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright (c) 2021, Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * 3. Neither the name of the Intel Corporation nor the names of its
16 * contributors may be used to endorse or promote products derived from
17 * this software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31 /*$FreeBSD$*/
32
33 /**
34 * @file if_ice_iflib.c
35 * @brief iflib driver implementation
36 *
37 * Contains the main entry point for the iflib driver implementation. It
38 * implements the various ifdi driver methods, and sets up the module and
39 * driver values to load an iflib driver.
40 */
41
42 #include "ice_iflib.h"
43 #include "ice_drv_info.h"
44 #include "ice_switch.h"
45 #include "ice_sched.h"
46
47 #include <sys/module.h>
48 #include <sys/sockio.h>
49 #include <sys/smp.h>
50 #include <dev/pci/pcivar.h>
51 #include <dev/pci/pcireg.h>
52
53 /*
54 * Device method prototypes
55 */
56
57 static void *ice_register(device_t);
58 static int ice_if_attach_pre(if_ctx_t);
59 static int ice_attach_pre_recovery_mode(struct ice_softc *sc);
60 static int ice_if_attach_post(if_ctx_t);
61 static void ice_attach_post_recovery_mode(struct ice_softc *sc);
62 static int ice_if_detach(if_ctx_t);
63 static int ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets);
64 static int ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets);
65 static int ice_if_msix_intr_assign(if_ctx_t ctx, int msix);
66 static void ice_if_queues_free(if_ctx_t ctx);
67 static int ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
68 static void ice_if_intr_enable(if_ctx_t ctx);
69 static void ice_if_intr_disable(if_ctx_t ctx);
70 static int ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid);
71 static int ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid);
72 static int ice_if_promisc_set(if_ctx_t ctx, int flags);
73 static void ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr);
74 static int ice_if_media_change(if_ctx_t ctx);
75 static void ice_if_init(if_ctx_t ctx);
76 static void ice_if_timer(if_ctx_t ctx, uint16_t qid);
77 static void ice_if_update_admin_status(if_ctx_t ctx);
78 static void ice_if_multi_set(if_ctx_t ctx);
79 static void ice_if_vlan_register(if_ctx_t ctx, u16 vtag);
80 static void ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag);
81 static void ice_if_stop(if_ctx_t ctx);
82 static uint64_t ice_if_get_counter(if_ctx_t ctx, ift_counter counter);
83 static int ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data);
84 static int ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req);
85 static int ice_if_suspend(if_ctx_t ctx);
86 static int ice_if_resume(if_ctx_t ctx);
87
88 static int ice_msix_que(void *arg);
89 static int ice_msix_admin(void *arg);
90
91 /*
92 * Helper function prototypes
93 */
94 static int ice_pci_mapping(struct ice_softc *sc);
95 static void ice_free_pci_mapping(struct ice_softc *sc);
96 static void ice_update_link_status(struct ice_softc *sc, bool update_media);
97 static void ice_init_device_features(struct ice_softc *sc);
98 static void ice_init_tx_tracking(struct ice_vsi *vsi);
99 static void ice_handle_reset_event(struct ice_softc *sc);
100 static void ice_handle_pf_reset_request(struct ice_softc *sc);
101 static void ice_prepare_for_reset(struct ice_softc *sc);
102 static int ice_rebuild_pf_vsi_qmap(struct ice_softc *sc);
103 static void ice_rebuild(struct ice_softc *sc);
104 static void ice_rebuild_recovery_mode(struct ice_softc *sc);
105 static void ice_free_irqvs(struct ice_softc *sc);
106 static void ice_update_rx_mbuf_sz(struct ice_softc *sc);
107 static void ice_poll_for_media_avail(struct ice_softc *sc);
108 static void ice_setup_scctx(struct ice_softc *sc);
109 static int ice_allocate_msix(struct ice_softc *sc);
110 static void ice_admin_timer(void *arg);
111 static void ice_transition_recovery_mode(struct ice_softc *sc);
112 static void ice_transition_safe_mode(struct ice_softc *sc);
113
114 /*
115 * Device Interface Declaration
116 */
117
118 /**
119 * @var ice_methods
120 * @brief ice driver method entry points
121 *
122 * List of device methods implementing the generic device interface used by
123 * the device stack to interact with the ice driver. Since this is an iflib
124 * driver, most of the methods point to the generic iflib implementation.
125 */
126 static device_method_t ice_methods[] = {
127 /* Device interface */
128 DEVMETHOD(device_register, ice_register),
129 DEVMETHOD(device_probe, iflib_device_probe_vendor),
130 DEVMETHOD(device_attach, iflib_device_attach),
131 DEVMETHOD(device_detach, iflib_device_detach),
132 DEVMETHOD(device_shutdown, iflib_device_shutdown),
133 DEVMETHOD(device_suspend, iflib_device_suspend),
134 DEVMETHOD(device_resume, iflib_device_resume),
135 DEVMETHOD_END
136 };
137
138 /**
139 * @var ice_iflib_methods
140 * @brief iflib method entry points
141 *
142 * List of device methods used by the iflib stack to interact with this
143 * driver. These are the real main entry points used to interact with this
144 * driver.
145 */
146 static device_method_t ice_iflib_methods[] = {
147 DEVMETHOD(ifdi_attach_pre, ice_if_attach_pre),
148 DEVMETHOD(ifdi_attach_post, ice_if_attach_post),
149 DEVMETHOD(ifdi_detach, ice_if_detach),
150 DEVMETHOD(ifdi_tx_queues_alloc, ice_if_tx_queues_alloc),
151 DEVMETHOD(ifdi_rx_queues_alloc, ice_if_rx_queues_alloc),
152 DEVMETHOD(ifdi_msix_intr_assign, ice_if_msix_intr_assign),
153 DEVMETHOD(ifdi_queues_free, ice_if_queues_free),
154 DEVMETHOD(ifdi_mtu_set, ice_if_mtu_set),
155 DEVMETHOD(ifdi_intr_enable, ice_if_intr_enable),
156 DEVMETHOD(ifdi_intr_disable, ice_if_intr_disable),
157 DEVMETHOD(ifdi_rx_queue_intr_enable, ice_if_rx_queue_intr_enable),
158 DEVMETHOD(ifdi_tx_queue_intr_enable, ice_if_tx_queue_intr_enable),
159 DEVMETHOD(ifdi_promisc_set, ice_if_promisc_set),
160 DEVMETHOD(ifdi_media_status, ice_if_media_status),
161 DEVMETHOD(ifdi_media_change, ice_if_media_change),
162 DEVMETHOD(ifdi_init, ice_if_init),
163 DEVMETHOD(ifdi_stop, ice_if_stop),
164 DEVMETHOD(ifdi_timer, ice_if_timer),
165 DEVMETHOD(ifdi_update_admin_status, ice_if_update_admin_status),
166 DEVMETHOD(ifdi_multi_set, ice_if_multi_set),
167 DEVMETHOD(ifdi_vlan_register, ice_if_vlan_register),
168 DEVMETHOD(ifdi_vlan_unregister, ice_if_vlan_unregister),
169 DEVMETHOD(ifdi_get_counter, ice_if_get_counter),
170 DEVMETHOD(ifdi_priv_ioctl, ice_if_priv_ioctl),
171 DEVMETHOD(ifdi_i2c_req, ice_if_i2c_req),
172 DEVMETHOD(ifdi_suspend, ice_if_suspend),
173 DEVMETHOD(ifdi_resume, ice_if_resume),
174 DEVMETHOD_END
175 };
176
177 /**
178 * @var ice_driver
179 * @brief driver structure for the generic device stack
180 *
181 * driver_t definition used to setup the generic device methods.
182 */
183 static driver_t ice_driver = {
184 .name = "ice",
185 .methods = ice_methods,
186 .size = sizeof(struct ice_softc),
187 };
188
189 /**
190 * @var ice_iflib_driver
191 * @brief driver structure for the iflib stack
192 *
193 * driver_t definition used to setup the iflib device methods.
194 */
195 static driver_t ice_iflib_driver = {
196 .name = "ice",
197 .methods = ice_iflib_methods,
198 .size = sizeof(struct ice_softc),
199 };
200
201 extern struct if_txrx ice_txrx;
202 extern struct if_txrx ice_recovery_txrx;
203
204 /**
205 * @var ice_sctx
206 * @brief ice driver shared context
207 *
208 * Structure defining shared values (context) that is used by all instances of
209 * the device. Primarily used to setup details about how the iflib stack
210 * should treat this driver. Also defines the default, minimum, and maximum
211 * number of descriptors in each ring.
212 */
213 static struct if_shared_ctx ice_sctx = {
214 .isc_magic = IFLIB_MAGIC,
215 .isc_q_align = PAGE_SIZE,
216
217 .isc_tx_maxsize = ICE_MAX_FRAME_SIZE,
218 /* We could technically set this as high as ICE_MAX_DMA_SEG_SIZE, but
219 * that doesn't make sense since that would be larger than the maximum
220 * size of a single packet.
221 */
222 .isc_tx_maxsegsize = ICE_MAX_FRAME_SIZE,
223
224 /* XXX: This is only used by iflib to ensure that
225 * scctx->isc_tx_tso_size_max + the VLAN header is a valid size.
226 */
227 .isc_tso_maxsize = ICE_TSO_SIZE + sizeof(struct ether_vlan_header),
228 /* XXX: This is used by iflib to set the number of segments in the TSO
229 * DMA tag. However, scctx->isc_tx_tso_segsize_max is used to set the
230 * related ifnet parameter.
231 */
232 .isc_tso_maxsegsize = ICE_MAX_DMA_SEG_SIZE,
233
234 .isc_rx_maxsize = ICE_MAX_FRAME_SIZE,
235 .isc_rx_nsegments = ICE_MAX_RX_SEGS,
236 .isc_rx_maxsegsize = ICE_MAX_FRAME_SIZE,
237
238 .isc_nfl = 1,
239 .isc_ntxqs = 1,
240 .isc_nrxqs = 1,
241
242 .isc_admin_intrcnt = 1,
243 .isc_vendor_info = ice_vendor_info_array,
244 .isc_driver_version = __DECONST(char *, ice_driver_version),
245 .isc_driver = &ice_iflib_driver,
246
247 /*
248 * IFLIB_NEED_SCRATCH ensures that mbufs have scratch space available
249 * for hardware checksum offload
250 *
251 * IFLIB_TSO_INIT_IP ensures that the TSO packets have zeroed out the
252 * IP sum field, required by our hardware to calculate valid TSO
253 * checksums.
254 *
255 * IFLIB_ADMIN_ALWAYS_RUN ensures that the administrative task runs
256 * even when the interface is down.
257 *
258 * IFLIB_SKIP_MSIX allows the driver to handle allocating MSI-X
259 * vectors manually instead of relying on iflib code to do this.
260 */
261 .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP |
262 IFLIB_ADMIN_ALWAYS_RUN | IFLIB_SKIP_MSIX,
263
264 .isc_nrxd_min = {ICE_MIN_DESC_COUNT},
265 .isc_ntxd_min = {ICE_MIN_DESC_COUNT},
266 .isc_nrxd_max = {ICE_IFLIB_MAX_DESC_COUNT},
267 .isc_ntxd_max = {ICE_IFLIB_MAX_DESC_COUNT},
268 .isc_nrxd_default = {ICE_DEFAULT_DESC_COUNT},
269 .isc_ntxd_default = {ICE_DEFAULT_DESC_COUNT},
270 };
271
272 /**
273 * @var ice_devclass
274 * @brief ice driver device class
275 *
276 * device class used to setup the ice driver module kobject class.
277 */
278 devclass_t ice_devclass;
279 DRIVER_MODULE(ice, pci, ice_driver, ice_devclass, ice_module_event_handler, 0);
280
281 MODULE_VERSION(ice, 1);
282 MODULE_DEPEND(ice, pci, 1, 1, 1);
283 MODULE_DEPEND(ice, ether, 1, 1, 1);
284 MODULE_DEPEND(ice, iflib, 1, 1, 1);
285
286 IFLIB_PNP_INFO(pci, ice, ice_vendor_info_array);
287
288 /* Static driver-wide sysctls */
289 #include "ice_iflib_sysctls.h"
290
291 /**
292 * ice_pci_mapping - Map PCI BAR memory
293 * @sc: device private softc
294 *
295 * Map PCI BAR 0 for device operation.
296 */
297 static int
ice_pci_mapping(struct ice_softc * sc)298 ice_pci_mapping(struct ice_softc *sc)
299 {
300 int rc;
301
302 /* Map BAR0 */
303 rc = ice_map_bar(sc->dev, &sc->bar0, 0);
304 if (rc)
305 return rc;
306
307 return 0;
308 }
309
310 /**
311 * ice_free_pci_mapping - Release PCI BAR memory
312 * @sc: device private softc
313 *
314 * Release PCI BARs which were previously mapped by ice_pci_mapping().
315 */
316 static void
ice_free_pci_mapping(struct ice_softc * sc)317 ice_free_pci_mapping(struct ice_softc *sc)
318 {
319 /* Free BAR0 */
320 ice_free_bar(sc->dev, &sc->bar0);
321 }
322
323 /*
324 * Device methods
325 */
326
327 /**
328 * ice_register - register device method callback
329 * @dev: the device being registered
330 *
331 * Returns a pointer to the shared context structure, which is used by iflib.
332 */
333 static void *
ice_register(device_t dev __unused)334 ice_register(device_t dev __unused)
335 {
336 return &ice_sctx;
337 } /* ice_register */
338
339 /**
340 * ice_setup_scctx - Setup the iflib softc context structure
341 * @sc: the device private structure
342 *
343 * Setup the parameters in if_softc_ctx_t structure used by the iflib stack
344 * when loading.
345 */
346 static void
ice_setup_scctx(struct ice_softc * sc)347 ice_setup_scctx(struct ice_softc *sc)
348 {
349 if_softc_ctx_t scctx = sc->scctx;
350 struct ice_hw *hw = &sc->hw;
351 bool safe_mode, recovery_mode;
352
353 safe_mode = ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE);
354 recovery_mode = ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE);
355
356 /*
357 * If the driver loads in Safe mode or Recovery mode, limit iflib to
358 * a single queue pair.
359 */
360 if (safe_mode || recovery_mode) {
361 scctx->isc_ntxqsets = scctx->isc_nrxqsets = 1;
362 scctx->isc_ntxqsets_max = 1;
363 scctx->isc_nrxqsets_max = 1;
364 } else {
365 /*
366 * iflib initially sets the isc_ntxqsets and isc_nrxqsets to
367 * the values of the override sysctls. Cache these initial
368 * values so that the driver can be aware of what the iflib
369 * sysctl value is when setting up MSI-X vectors.
370 */
371 sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets;
372 sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets;
373
374 if (scctx->isc_ntxqsets == 0)
375 scctx->isc_ntxqsets = hw->func_caps.common_cap.rss_table_size;
376 if (scctx->isc_nrxqsets == 0)
377 scctx->isc_nrxqsets = hw->func_caps.common_cap.rss_table_size;
378
379 scctx->isc_ntxqsets_max = hw->func_caps.common_cap.num_txq;
380 scctx->isc_nrxqsets_max = hw->func_caps.common_cap.num_rxq;
381
382 /*
383 * Sanity check that the iflib sysctl values are within the
384 * maximum supported range.
385 */
386 if (sc->ifc_sysctl_ntxqs > scctx->isc_ntxqsets_max)
387 sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets_max;
388 if (sc->ifc_sysctl_nrxqs > scctx->isc_nrxqsets_max)
389 sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets_max;
390 }
391
392 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0]
393 * sizeof(struct ice_tx_desc), DBA_ALIGN);
394 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0]
395 * sizeof(union ice_32b_rx_flex_desc), DBA_ALIGN);
396
397 scctx->isc_tx_nsegments = ICE_MAX_TX_SEGS;
398 scctx->isc_tx_tso_segments_max = ICE_MAX_TSO_SEGS;
399 scctx->isc_tx_tso_size_max = ICE_TSO_SIZE;
400 scctx->isc_tx_tso_segsize_max = ICE_MAX_DMA_SEG_SIZE;
401
402 scctx->isc_msix_bar = PCIR_BAR(ICE_MSIX_BAR);
403 scctx->isc_rss_table_size = hw->func_caps.common_cap.rss_table_size;
404
405 /*
406 * If the driver loads in recovery mode, disable Tx/Rx functionality
407 */
408 if (recovery_mode)
409 scctx->isc_txrx = &ice_recovery_txrx;
410 else
411 scctx->isc_txrx = &ice_txrx;
412
413 /*
414 * If the driver loads in Safe mode or Recovery mode, disable
415 * advanced features including hardware offloads.
416 */
417 if (safe_mode || recovery_mode) {
418 scctx->isc_capenable = ICE_SAFE_CAPS;
419 scctx->isc_tx_csum_flags = 0;
420 } else {
421 scctx->isc_capenable = ICE_FULL_CAPS;
422 scctx->isc_tx_csum_flags = ICE_CSUM_OFFLOAD;
423 }
424
425 scctx->isc_capabilities = scctx->isc_capenable;
426 } /* ice_setup_scctx */
427
428 /**
429 * ice_if_attach_pre - Early device attach logic
430 * @ctx: the iflib context structure
431 *
432 * Called by iflib during the attach process. Earliest main driver entry
433 * point which performs necessary hardware and driver initialization. Called
434 * before the Tx and Rx queues are allocated.
435 */
436 static int
ice_if_attach_pre(if_ctx_t ctx)437 ice_if_attach_pre(if_ctx_t ctx)
438 {
439 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
440 enum ice_fw_modes fw_mode;
441 enum ice_status status;
442 if_softc_ctx_t scctx;
443 struct ice_hw *hw;
444 device_t dev;
445 int err;
446
447 device_printf(iflib_get_dev(ctx), "Loading the iflib ice driver\n");
448
449 ice_set_state(&sc->state, ICE_STATE_ATTACHING);
450
451 sc->ctx = ctx;
452 sc->media = iflib_get_media(ctx);
453 sc->sctx = iflib_get_sctx(ctx);
454 sc->iflib_ctx_lock = iflib_ctx_lock_get(ctx);
455
456 dev = sc->dev = iflib_get_dev(ctx);
457 scctx = sc->scctx = iflib_get_softc_ctx(ctx);
458
459 hw = &sc->hw;
460 hw->back = sc;
461
462 snprintf(sc->admin_mtx_name, sizeof(sc->admin_mtx_name),
463 "%s:admin", device_get_nameunit(dev));
464 mtx_init(&sc->admin_mtx, sc->admin_mtx_name, NULL, MTX_DEF);
465 callout_init_mtx(&sc->admin_timer, &sc->admin_mtx, 0);
466
467 ASSERT_CTX_LOCKED(sc);
468
469 if (ice_pci_mapping(sc)) {
470 err = (ENXIO);
471 goto destroy_admin_timer;
472 }
473
474 /* Save off the PCI information */
475 ice_save_pci_info(hw, dev);
476
477 /* create tunables as early as possible */
478 ice_add_device_tunables(sc);
479
480 /* Setup ControlQ lengths */
481 ice_set_ctrlq_len(hw);
482
483 fw_mode = ice_get_fw_mode(hw);
484 if (fw_mode == ICE_FW_MODE_REC) {
485 device_printf(dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
486
487 err = ice_attach_pre_recovery_mode(sc);
488 if (err)
489 goto free_pci_mapping;
490
491 return (0);
492 }
493
494 /* Initialize the hw data structure */
495 status = ice_init_hw(hw);
496 if (status) {
497 if (status == ICE_ERR_FW_API_VER) {
498 /* Enter recovery mode, so that the driver remains
499 * loaded. This way, if the system administrator
500 * cannot update the driver, they may still attempt to
501 * downgrade the NVM.
502 */
503 err = ice_attach_pre_recovery_mode(sc);
504 if (err)
505 goto free_pci_mapping;
506
507 return (0);
508 } else {
509 err = EIO;
510 device_printf(dev, "Unable to initialize hw, err %s aq_err %s\n",
511 ice_status_str(status),
512 ice_aq_str(hw->adminq.sq_last_status));
513 }
514 goto free_pci_mapping;
515 }
516
517 /* Notify firmware of the device driver version */
518 err = ice_send_version(sc);
519 if (err)
520 goto deinit_hw;
521
522 ice_load_pkg_file(sc);
523
524 err = ice_init_link_events(sc);
525 if (err) {
526 device_printf(dev, "ice_init_link_events failed: %s\n",
527 ice_err_str(err));
528 goto deinit_hw;
529 }
530
531 ice_print_nvm_version(sc);
532
533 ice_init_device_features(sc);
534
535 /* Setup the MAC address */
536 iflib_set_mac(ctx, hw->port_info->mac.lan_addr);
537
538 /* Setup the iflib softc context structure */
539 ice_setup_scctx(sc);
540
541 /* Initialize the Tx queue manager */
542 err = ice_resmgr_init(&sc->tx_qmgr, hw->func_caps.common_cap.num_txq);
543 if (err) {
544 device_printf(dev, "Unable to initialize Tx queue manager: %s\n",
545 ice_err_str(err));
546 goto deinit_hw;
547 }
548
549 /* Initialize the Rx queue manager */
550 err = ice_resmgr_init(&sc->rx_qmgr, hw->func_caps.common_cap.num_rxq);
551 if (err) {
552 device_printf(dev, "Unable to initialize Rx queue manager: %s\n",
553 ice_err_str(err));
554 goto free_tx_qmgr;
555 }
556
557 /* Initialize the interrupt resource manager */
558 err = ice_alloc_intr_tracking(sc);
559 if (err)
560 /* Errors are already printed */
561 goto free_rx_qmgr;
562
563 /* Determine maximum number of VSIs we'll prepare for */
564 sc->num_available_vsi = min(ICE_MAX_VSI_AVAILABLE,
565 hw->func_caps.guar_num_vsi);
566
567 if (!sc->num_available_vsi) {
568 err = EIO;
569 device_printf(dev, "No VSIs allocated to host\n");
570 goto free_intr_tracking;
571 }
572
573 /* Allocate storage for the VSI pointers */
574 sc->all_vsi = (struct ice_vsi **)
575 malloc(sizeof(struct ice_vsi *) * sc->num_available_vsi,
576 M_ICE, M_WAITOK | M_ZERO);
577 if (!sc->all_vsi) {
578 err = ENOMEM;
579 device_printf(dev, "Unable to allocate VSI array\n");
580 goto free_intr_tracking;
581 }
582
583 /*
584 * Prepare the statically allocated primary PF VSI in the softc
585 * structure. Other VSIs will be dynamically allocated as needed.
586 */
587 ice_setup_pf_vsi(sc);
588
589 err = ice_alloc_vsi_qmap(&sc->pf_vsi, scctx->isc_ntxqsets_max,
590 scctx->isc_nrxqsets_max);
591 if (err) {
592 device_printf(dev, "Unable to allocate VSI Queue maps\n");
593 goto free_main_vsi;
594 }
595
596 /* Allocate MSI-X vectors (due to isc_flags IFLIB_SKIP_MSIX) */
597 err = ice_allocate_msix(sc);
598 if (err)
599 goto free_main_vsi;
600
601 return 0;
602
603 free_main_vsi:
604 /* ice_release_vsi will free the queue maps if they were allocated */
605 ice_release_vsi(&sc->pf_vsi);
606 free(sc->all_vsi, M_ICE);
607 sc->all_vsi = NULL;
608 free_intr_tracking:
609 ice_free_intr_tracking(sc);
610 free_rx_qmgr:
611 ice_resmgr_destroy(&sc->rx_qmgr);
612 free_tx_qmgr:
613 ice_resmgr_destroy(&sc->tx_qmgr);
614 deinit_hw:
615 ice_deinit_hw(hw);
616 free_pci_mapping:
617 ice_free_pci_mapping(sc);
618 destroy_admin_timer:
619 mtx_lock(&sc->admin_mtx);
620 callout_stop(&sc->admin_timer);
621 mtx_unlock(&sc->admin_mtx);
622 mtx_destroy(&sc->admin_mtx);
623 return err;
624 } /* ice_if_attach_pre */
625
626 /**
627 * ice_attach_pre_recovery_mode - Limited driver attach_pre for FW recovery
628 * @sc: the device private softc
629 *
630 * Loads the device driver in limited Firmware Recovery mode, intended to
631 * allow users to update the firmware to attempt to recover the device.
632 *
633 * @remark We may enter recovery mode in case either (a) the firmware is
634 * detected to be in an invalid state and must be re-programmed, or (b) the
635 * driver detects that the loaded firmware has a non-compatible API version
636 * that the driver cannot operate with.
637 */
638 static int
ice_attach_pre_recovery_mode(struct ice_softc * sc)639 ice_attach_pre_recovery_mode(struct ice_softc *sc)
640 {
641 ice_set_state(&sc->state, ICE_STATE_RECOVERY_MODE);
642
643 /* Setup the iflib softc context */
644 ice_setup_scctx(sc);
645
646 /* Setup the PF VSI back pointer */
647 sc->pf_vsi.sc = sc;
648
649 /*
650 * We still need to allocate MSI-X vectors since we need one vector to
651 * run the administrative admin interrupt
652 */
653 return ice_allocate_msix(sc);
654 }
655
656 /**
657 * ice_update_link_status - notify OS of link state change
658 * @sc: device private softc structure
659 * @update_media: true if we should update media even if link didn't change
660 *
661 * Called to notify iflib core of link status changes. Should be called once
662 * during attach_post, and whenever link status changes during runtime.
663 *
664 * This call only updates the currently supported media types if the link
665 * status changed, or if update_media is set to true.
666 */
667 static void
ice_update_link_status(struct ice_softc * sc,bool update_media)668 ice_update_link_status(struct ice_softc *sc, bool update_media)
669 {
670 struct ice_hw *hw = &sc->hw;
671 enum ice_status status;
672
673 /* Never report link up when in recovery mode */
674 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
675 return;
676
677 /* Report link status to iflib only once each time it changes */
678 if (!ice_testandset_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED)) {
679 if (sc->link_up) { /* link is up */
680 uint64_t baudrate = ice_aq_speed_to_rate(sc->hw.port_info);
681
682 ice_set_default_local_lldp_mib(sc);
683
684 iflib_link_state_change(sc->ctx, LINK_STATE_UP, baudrate);
685
686 ice_link_up_msg(sc);
687
688 update_media = true;
689 } else { /* link is down */
690 iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0);
691
692 update_media = true;
693 }
694 }
695
696 /* Update the supported media types */
697 if (update_media) {
698 status = ice_add_media_types(sc, sc->media);
699 if (status)
700 device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n",
701 ice_status_str(status),
702 ice_aq_str(hw->adminq.sq_last_status));
703 }
704
705 /* TODO: notify VFs of link state change */
706 }
707
708 /**
709 * ice_if_attach_post - Late device attach logic
710 * @ctx: the iflib context structure
711 *
712 * Called by iflib to finish up attaching the device. Performs any attach
713 * logic which must wait until after the Tx and Rx queues have been
714 * allocated.
715 */
716 static int
ice_if_attach_post(if_ctx_t ctx)717 ice_if_attach_post(if_ctx_t ctx)
718 {
719 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
720 if_t ifp = iflib_get_ifp(ctx);
721 int err;
722
723 ASSERT_CTX_LOCKED(sc);
724
725 /* We don't yet support loading if MSI-X is not supported */
726 if (sc->scctx->isc_intr != IFLIB_INTR_MSIX) {
727 device_printf(sc->dev, "The ice driver does not support loading without MSI-X\n");
728 return (ENOTSUP);
729 }
730
731 /* The ifnet structure hasn't yet been initialized when the attach_pre
732 * handler is called, so wait until attach_post to setup the
733 * isc_max_frame_size.
734 */
735
736 sc->ifp = ifp;
737 sc->scctx->isc_max_frame_size = ifp->if_mtu +
738 ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN;
739
740 /*
741 * If we are in recovery mode, only perform a limited subset of
742 * initialization to support NVM recovery.
743 */
744 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
745 ice_attach_post_recovery_mode(sc);
746 return (0);
747 }
748
749 sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size;
750
751 err = ice_initialize_vsi(&sc->pf_vsi);
752 if (err) {
753 device_printf(sc->dev, "Unable to initialize Main VSI: %s\n",
754 ice_err_str(err));
755 return err;
756 }
757
758 /* Enable FW health event reporting */
759 ice_init_health_events(sc);
760
761 /* Configure the main PF VSI for RSS */
762 err = ice_config_rss(&sc->pf_vsi);
763 if (err) {
764 device_printf(sc->dev,
765 "Unable to configure RSS for the main VSI, err %s\n",
766 ice_err_str(err));
767 return err;
768 }
769
770 /* Configure switch to drop transmitted LLDP and PAUSE frames */
771 err = ice_cfg_pf_ethertype_filters(sc);
772 if (err)
773 return err;
774
775 ice_get_and_print_bus_info(sc);
776
777 ice_set_link_management_mode(sc);
778
779 ice_init_saved_phy_cfg(sc);
780
781 ice_cfg_pba_num(sc);
782
783 ice_add_device_sysctls(sc);
784
785 /* Get DCBX/LLDP state and start DCBX agent */
786 ice_init_dcb_setup(sc);
787
788 /* Setup link configuration parameters */
789 ice_init_link_configuration(sc);
790 ice_update_link_status(sc, true);
791
792 /* Configure interrupt causes for the administrative interrupt */
793 ice_configure_misc_interrupts(sc);
794
795 /* Enable ITR 0 right away, so that we can handle admin interrupts */
796 ice_enable_intr(&sc->hw, sc->irqvs[0].me);
797
798 /* Start the admin timer */
799 mtx_lock(&sc->admin_mtx);
800 callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc);
801 mtx_unlock(&sc->admin_mtx);
802
803 ice_clear_state(&sc->state, ICE_STATE_ATTACHING);
804
805 return 0;
806 } /* ice_if_attach_post */
807
808 /**
809 * ice_attach_post_recovery_mode - Limited driver attach_post for FW recovery
810 * @sc: the device private softc
811 *
812 * Performs minimal work to prepare the driver to recover an NVM in case the
813 * firmware is in recovery mode.
814 */
815 static void
ice_attach_post_recovery_mode(struct ice_softc * sc)816 ice_attach_post_recovery_mode(struct ice_softc *sc)
817 {
818 /* Configure interrupt causes for the administrative interrupt */
819 ice_configure_misc_interrupts(sc);
820
821 /* Enable ITR 0 right away, so that we can handle admin interrupts */
822 ice_enable_intr(&sc->hw, sc->irqvs[0].me);
823
824 /* Start the admin timer */
825 mtx_lock(&sc->admin_mtx);
826 callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc);
827 mtx_unlock(&sc->admin_mtx);
828
829 ice_clear_state(&sc->state, ICE_STATE_ATTACHING);
830 }
831
832 /**
833 * ice_free_irqvs - Free IRQ vector memory
834 * @sc: the device private softc structure
835 *
836 * Free IRQ vector memory allocated during ice_if_msix_intr_assign.
837 */
838 static void
ice_free_irqvs(struct ice_softc * sc)839 ice_free_irqvs(struct ice_softc *sc)
840 {
841 struct ice_vsi *vsi = &sc->pf_vsi;
842 if_ctx_t ctx = sc->ctx;
843 int i;
844
845 /* If the irqvs array is NULL, then there are no vectors to free */
846 if (sc->irqvs == NULL)
847 return;
848
849 /* Free the IRQ vectors */
850 for (i = 0; i < sc->num_irq_vectors; i++)
851 iflib_irq_free(ctx, &sc->irqvs[i].irq);
852
853 /* Clear the irqv pointers */
854 for (i = 0; i < vsi->num_rx_queues; i++)
855 vsi->rx_queues[i].irqv = NULL;
856
857 for (i = 0; i < vsi->num_tx_queues; i++)
858 vsi->tx_queues[i].irqv = NULL;
859
860 /* Release the vector array memory */
861 free(sc->irqvs, M_ICE);
862 sc->irqvs = NULL;
863 sc->num_irq_vectors = 0;
864 }
865
866 /**
867 * ice_if_detach - Device driver detach logic
868 * @ctx: iflib context structure
869 *
870 * Perform device shutdown logic to detach the device driver.
871 *
872 * Note that there is no guarantee of the ordering of ice_if_queues_free() and
873 * ice_if_detach(). It is possible for the functions to be called in either
874 * order, and they must not assume to have a strict ordering.
875 */
876 static int
ice_if_detach(if_ctx_t ctx)877 ice_if_detach(if_ctx_t ctx)
878 {
879 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
880 struct ice_vsi *vsi = &sc->pf_vsi;
881 int i;
882
883 ASSERT_CTX_LOCKED(sc);
884
885 /* Indicate that we're detaching */
886 ice_set_state(&sc->state, ICE_STATE_DETACHING);
887
888 /* Stop the admin timer */
889 mtx_lock(&sc->admin_mtx);
890 callout_stop(&sc->admin_timer);
891 mtx_unlock(&sc->admin_mtx);
892 mtx_destroy(&sc->admin_mtx);
893
894 /* Free allocated media types */
895 ifmedia_removeall(sc->media);
896
897 /* Free the Tx and Rx sysctl contexts, and assign NULL to the node
898 * pointers. Note, the calls here and those in ice_if_queues_free()
899 * are *BOTH* necessary, as we cannot guarantee which path will be
900 * run first
901 */
902 ice_vsi_del_txqs_ctx(vsi);
903 ice_vsi_del_rxqs_ctx(vsi);
904
905 /* Release MSI-X resources */
906 ice_free_irqvs(sc);
907
908 for (i = 0; i < sc->num_available_vsi; i++) {
909 if (sc->all_vsi[i])
910 ice_release_vsi(sc->all_vsi[i]);
911 }
912
913 if (sc->all_vsi) {
914 free(sc->all_vsi, M_ICE);
915 sc->all_vsi = NULL;
916 }
917
918 /* Release MSI-X memory */
919 pci_release_msi(sc->dev);
920
921 if (sc->msix_table != NULL) {
922 bus_release_resource(sc->dev, SYS_RES_MEMORY,
923 rman_get_rid(sc->msix_table),
924 sc->msix_table);
925 sc->msix_table = NULL;
926 }
927
928 ice_free_intr_tracking(sc);
929
930 /* Destroy the queue managers */
931 ice_resmgr_destroy(&sc->tx_qmgr);
932 ice_resmgr_destroy(&sc->rx_qmgr);
933
934 if (!ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
935 ice_deinit_hw(&sc->hw);
936
937 ice_free_pci_mapping(sc);
938
939 return 0;
940 } /* ice_if_detach */
941
942 /**
943 * ice_if_tx_queues_alloc - Allocate Tx queue memory
944 * @ctx: iflib context structure
945 * @vaddrs: virtual addresses for the queue memory
946 * @paddrs: physical addresses for the queue memory
947 * @ntxqs: the number of Tx queues per set (should always be 1)
948 * @ntxqsets: the number of Tx queue sets to allocate
949 *
950 * Called by iflib to allocate Tx queues for the device. Allocates driver
951 * memory to track each queue, the status arrays used for descriptor
952 * status reporting, and Tx queue sysctls.
953 */
954 static int
ice_if_tx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int __invariant_only ntxqs,int ntxqsets)955 ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
956 int __invariant_only ntxqs, int ntxqsets)
957 {
958 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
959 struct ice_vsi *vsi = &sc->pf_vsi;
960 struct ice_tx_queue *txq;
961 int err, i, j;
962
963 MPASS(ntxqs == 1);
964 MPASS(sc->scctx->isc_ntxd[0] <= ICE_MAX_DESC_COUNT);
965 ASSERT_CTX_LOCKED(sc);
966
967 /* Do not bother allocating queues if we're in recovery mode */
968 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
969 return (0);
970
971 /* Allocate queue structure memory */
972 if (!(vsi->tx_queues =
973 (struct ice_tx_queue *) malloc(sizeof(struct ice_tx_queue) * ntxqsets, M_ICE, M_WAITOK | M_ZERO))) {
974 device_printf(sc->dev, "Unable to allocate Tx queue memory\n");
975 return (ENOMEM);
976 }
977
978 /* Allocate report status arrays */
979 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
980 if (!(txq->tx_rsq =
981 (uint16_t *) malloc(sizeof(uint16_t) * sc->scctx->isc_ntxd[0], M_ICE, M_WAITOK))) {
982 device_printf(sc->dev, "Unable to allocate tx_rsq memory\n");
983 err = ENOMEM;
984 goto free_tx_queues;
985 }
986 /* Initialize report status array */
987 for (j = 0; j < sc->scctx->isc_ntxd[0]; j++)
988 txq->tx_rsq[j] = QIDX_INVALID;
989 }
990
991 /* Assign queues from PF space to the main VSI */
992 err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, ntxqsets);
993 if (err) {
994 device_printf(sc->dev, "Unable to assign PF queues: %s\n",
995 ice_err_str(err));
996 goto free_tx_queues;
997 }
998 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
999
1000 /* Add Tx queue sysctls context */
1001 ice_vsi_add_txqs_ctx(vsi);
1002
1003 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
1004 /* q_handle == me when only one TC */
1005 txq->me = txq->q_handle = i;
1006 txq->vsi = vsi;
1007
1008 /* store the queue size for easier access */
1009 txq->desc_count = sc->scctx->isc_ntxd[0];
1010
1011 /* get the virtual and physical address of the hardware queues */
1012 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]);
1013 txq->tx_base = (struct ice_tx_desc *)vaddrs[i];
1014 txq->tx_paddr = paddrs[i];
1015
1016 ice_add_txq_sysctls(txq);
1017 }
1018
1019 vsi->num_tx_queues = ntxqsets;
1020
1021 return (0);
1022
1023 free_tx_queues:
1024 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
1025 if (txq->tx_rsq != NULL) {
1026 free(txq->tx_rsq, M_ICE);
1027 txq->tx_rsq = NULL;
1028 }
1029 }
1030 free(vsi->tx_queues, M_ICE);
1031 vsi->tx_queues = NULL;
1032 return err;
1033 }
1034
1035 /**
1036 * ice_if_rx_queues_alloc - Allocate Rx queue memory
1037 * @ctx: iflib context structure
1038 * @vaddrs: virtual addresses for the queue memory
1039 * @paddrs: physical addresses for the queue memory
1040 * @nrxqs: number of Rx queues per set (should always be 1)
1041 * @nrxqsets: number of Rx queue sets to allocate
1042 *
1043 * Called by iflib to allocate Rx queues for the device. Allocates driver
1044 * memory to track each queue, as well as sets up the Rx queue sysctls.
1045 */
1046 static int
ice_if_rx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int __invariant_only nrxqs,int nrxqsets)1047 ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
1048 int __invariant_only nrxqs, int nrxqsets)
1049 {
1050 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1051 struct ice_vsi *vsi = &sc->pf_vsi;
1052 struct ice_rx_queue *rxq;
1053 int err, i;
1054
1055 MPASS(nrxqs == 1);
1056 MPASS(sc->scctx->isc_nrxd[0] <= ICE_MAX_DESC_COUNT);
1057 ASSERT_CTX_LOCKED(sc);
1058
1059 /* Do not bother allocating queues if we're in recovery mode */
1060 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1061 return (0);
1062
1063 /* Allocate queue structure memory */
1064 if (!(vsi->rx_queues =
1065 (struct ice_rx_queue *) malloc(sizeof(struct ice_rx_queue) * nrxqsets, M_ICE, M_WAITOK | M_ZERO))) {
1066 device_printf(sc->dev, "Unable to allocate Rx queue memory\n");
1067 return (ENOMEM);
1068 }
1069
1070 /* Assign queues from PF space to the main VSI */
1071 err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, nrxqsets);
1072 if (err) {
1073 device_printf(sc->dev, "Unable to assign PF queues: %s\n",
1074 ice_err_str(err));
1075 goto free_rx_queues;
1076 }
1077 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
1078
1079 /* Add Rx queue sysctls context */
1080 ice_vsi_add_rxqs_ctx(vsi);
1081
1082 for (i = 0, rxq = vsi->rx_queues; i < nrxqsets; i++, rxq++) {
1083 rxq->me = i;
1084 rxq->vsi = vsi;
1085
1086 /* store the queue size for easier access */
1087 rxq->desc_count = sc->scctx->isc_nrxd[0];
1088
1089 /* get the virtual and physical address of the hardware queues */
1090 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]);
1091 rxq->rx_base = (union ice_32b_rx_flex_desc *)vaddrs[i];
1092 rxq->rx_paddr = paddrs[i];
1093
1094 ice_add_rxq_sysctls(rxq);
1095 }
1096
1097 vsi->num_rx_queues = nrxqsets;
1098
1099 return (0);
1100
1101 free_rx_queues:
1102 free(vsi->rx_queues, M_ICE);
1103 vsi->rx_queues = NULL;
1104 return err;
1105 }
1106
1107 /**
1108 * ice_if_queues_free - Free queue memory
1109 * @ctx: the iflib context structure
1110 *
1111 * Free queue memory allocated by ice_if_tx_queues_alloc() and
1112 * ice_if_rx_queues_alloc().
1113 *
1114 * There is no guarantee that ice_if_queues_free() and ice_if_detach() will be
1115 * called in the same order. It's possible for ice_if_queues_free() to be
1116 * called prior to ice_if_detach(), and vice versa.
1117 *
1118 * For this reason, the main VSI is a static member of the ice_softc, which is
1119 * not free'd until after iflib finishes calling both of these functions.
1120 *
1121 * Thus, care must be taken in how we manage the memory being freed by this
1122 * function, and in what tasks it can and must perform.
1123 */
1124 static void
ice_if_queues_free(if_ctx_t ctx)1125 ice_if_queues_free(if_ctx_t ctx)
1126 {
1127 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1128 struct ice_vsi *vsi = &sc->pf_vsi;
1129 struct ice_tx_queue *txq;
1130 int i;
1131
1132 /* Free the Tx and Rx sysctl contexts, and assign NULL to the node
1133 * pointers. Note, the calls here and those in ice_if_detach()
1134 * are *BOTH* necessary, as we cannot guarantee which path will be
1135 * run first
1136 */
1137 ice_vsi_del_txqs_ctx(vsi);
1138 ice_vsi_del_rxqs_ctx(vsi);
1139
1140 /* Release MSI-X IRQ vectors, if not yet released in ice_if_detach */
1141 ice_free_irqvs(sc);
1142
1143 if (vsi->tx_queues != NULL) {
1144 /* free the tx_rsq arrays */
1145 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) {
1146 if (txq->tx_rsq != NULL) {
1147 free(txq->tx_rsq, M_ICE);
1148 txq->tx_rsq = NULL;
1149 }
1150 }
1151 free(vsi->tx_queues, M_ICE);
1152 vsi->tx_queues = NULL;
1153 vsi->num_tx_queues = 0;
1154 }
1155 if (vsi->rx_queues != NULL) {
1156 free(vsi->rx_queues, M_ICE);
1157 vsi->rx_queues = NULL;
1158 vsi->num_rx_queues = 0;
1159 }
1160 }
1161
1162 /**
1163 * ice_msix_que - Fast interrupt handler for MSI-X receive queues
1164 * @arg: The Rx queue memory
1165 *
1166 * Interrupt filter function for iflib MSI-X interrupts. Called by iflib when
1167 * an MSI-X interrupt for a given queue is triggered. Currently this just asks
1168 * iflib to schedule the main Rx thread.
1169 */
1170 static int
ice_msix_que(void * arg)1171 ice_msix_que(void *arg)
1172 {
1173 struct ice_rx_queue __unused *rxq = (struct ice_rx_queue *)arg;
1174
1175 /* TODO: dynamic ITR algorithm?? */
1176
1177 return (FILTER_SCHEDULE_THREAD);
1178 }
1179
1180 /**
1181 * ice_msix_admin - Fast interrupt handler for MSI-X admin interrupt
1182 * @arg: pointer to device softc memory
1183 *
1184 * Called by iflib when an administrative interrupt occurs. Should perform any
1185 * fast logic for handling the interrupt cause, and then indicate whether the
1186 * admin task needs to be queued.
1187 */
1188 static int
ice_msix_admin(void * arg)1189 ice_msix_admin(void *arg)
1190 {
1191 struct ice_softc *sc = (struct ice_softc *)arg;
1192 struct ice_hw *hw = &sc->hw;
1193 device_t dev = sc->dev;
1194 u32 oicr;
1195
1196 /* There is no safe way to modify the enabled miscellaneous causes of
1197 * the OICR vector at runtime, as doing so would be prone to race
1198 * conditions. Reading PFINT_OICR will unmask the associated interrupt
1199 * causes and allow future interrupts to occur. The admin interrupt
1200 * vector will not be re-enabled until after we exit this function,
1201 * but any delayed tasks must be resilient against possible "late
1202 * arrival" interrupts that occur while we're already handling the
1203 * task. This is done by using state bits and serializing these
1204 * delayed tasks via the admin status task function.
1205 */
1206 oicr = rd32(hw, PFINT_OICR);
1207
1208 /* Processing multiple controlq interrupts on a single vector does not
1209 * provide an indication of which controlq triggered the interrupt.
1210 * We might try reading the INTEVENT bit of the respective PFINT_*_CTL
1211 * registers. However, the INTEVENT bit is not guaranteed to be set as
1212 * it gets automatically cleared when the hardware acknowledges the
1213 * interrupt.
1214 *
1215 * This means we don't really have a good indication of whether or
1216 * which controlq triggered this interrupt. We'll just notify the
1217 * admin task that it should check all the controlqs.
1218 */
1219 ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);
1220
1221 if (oicr & PFINT_OICR_VFLR_M) {
1222 ice_set_state(&sc->state, ICE_STATE_VFLR_PENDING);
1223 }
1224
1225 if (oicr & PFINT_OICR_MAL_DETECT_M) {
1226 ice_set_state(&sc->state, ICE_STATE_MDD_PENDING);
1227 }
1228
1229 if (oicr & PFINT_OICR_GRST_M) {
1230 u32 reset;
1231
1232 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
1233 GLGEN_RSTAT_RESET_TYPE_S;
1234
1235 if (reset == ICE_RESET_CORER)
1236 sc->soft_stats.corer_count++;
1237 else if (reset == ICE_RESET_GLOBR)
1238 sc->soft_stats.globr_count++;
1239 else
1240 sc->soft_stats.empr_count++;
1241
1242 /* There are a couple of bits at play for handling resets.
1243 * First, the ICE_STATE_RESET_OICR_RECV bit is used to
1244 * indicate that the driver has received an OICR with a reset
1245 * bit active, indicating that a CORER/GLOBR/EMPR is about to
1246 * happen. Second, we set hw->reset_ongoing to indicate that
1247 * the hardware is in reset. We will set this back to false as
1248 * soon as the driver has determined that the hardware is out
1249 * of reset.
1250 *
1251 * If the driver wishes to trigger a reqest, it can set one of
1252 * the ICE_STATE_RESET_*_REQ bits, which will trigger the
1253 * correct type of reset.
1254 */
1255 if (!ice_testandset_state(&sc->state, ICE_STATE_RESET_OICR_RECV))
1256 hw->reset_ongoing = true;
1257 }
1258
1259 if (oicr & PFINT_OICR_ECC_ERR_M) {
1260 device_printf(dev, "ECC Error detected!\n");
1261 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1262 }
1263
1264 if (oicr & PFINT_OICR_PE_CRITERR_M) {
1265 device_printf(dev, "Critical Protocol Engine Error detected!\n");
1266 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1267 }
1268
1269 if (oicr & PFINT_OICR_PCI_EXCEPTION_M) {
1270 device_printf(dev, "PCI Exception detected!\n");
1271 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1272 }
1273
1274 if (oicr & PFINT_OICR_HMC_ERR_M) {
1275 /* Log the HMC errors, but don't disable the interrupt cause */
1276 ice_log_hmc_error(hw, dev);
1277 }
1278
1279 return (FILTER_SCHEDULE_THREAD);
1280 }
1281
1282 /**
1283 * ice_allocate_msix - Allocate MSI-X vectors for the interface
1284 * @sc: the device private softc
1285 *
1286 * Map the MSI-X bar, and then request MSI-X vectors in a two-stage process.
1287 *
1288 * First, determine a suitable total number of vectors based on the number
1289 * of CPUs, RSS buckets, the administrative vector, and other demands such as
1290 * RDMA.
1291 *
1292 * Request the desired amount of vectors, and see how many we obtain. If we
1293 * don't obtain as many as desired, reduce the demands by lowering the number
1294 * of requested queues or reducing the demand from other features such as
1295 * RDMA.
1296 *
1297 * @remark This function is required because the driver sets the
1298 * IFLIB_SKIP_MSIX flag indicating that the driver will manage MSI-X vectors
1299 * manually.
1300 *
1301 * @remark This driver will only use MSI-X vectors. If this is not possible,
1302 * neither MSI or legacy interrupts will be tried.
1303 *
1304 * @post on success this function must set the following scctx parameters:
1305 * isc_vectors, isc_nrxqsets, isc_ntxqsets, and isc_intr.
1306 *
1307 * @returns zero on success or an error code on failure.
1308 */
1309 static int
ice_allocate_msix(struct ice_softc * sc)1310 ice_allocate_msix(struct ice_softc *sc)
1311 {
1312 bool iflib_override_queue_count = false;
1313 if_softc_ctx_t scctx = sc->scctx;
1314 device_t dev = sc->dev;
1315 cpuset_t cpus;
1316 int bar, queues, vectors, requested;
1317 int err = 0;
1318
1319 /* Allocate the MSI-X bar */
1320 bar = scctx->isc_msix_bar;
1321 sc->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE);
1322 if (!sc->msix_table) {
1323 device_printf(dev, "Unable to map MSI-X table\n");
1324 return (ENOMEM);
1325 }
1326
1327 /* Check if the iflib queue count sysctls have been set */
1328 if (sc->ifc_sysctl_ntxqs || sc->ifc_sysctl_nrxqs)
1329 iflib_override_queue_count = true;
1330
1331 err = bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus);
1332 if (err) {
1333 device_printf(dev, "%s: Unable to fetch the CPU list: %s\n",
1334 __func__, ice_err_str(err));
1335 CPU_COPY(&all_cpus, &cpus);
1336 }
1337
1338 /* Attempt to mimic behavior of iflib_msix_init */
1339 if (iflib_override_queue_count) {
1340 /*
1341 * If the override sysctls have been set, limit the queues to
1342 * the number of logical CPUs.
1343 */
1344 queues = mp_ncpus;
1345 } else {
1346 /*
1347 * Otherwise, limit the queue count to the CPUs associated
1348 * with the NUMA node the device is associated with.
1349 */
1350 queues = CPU_COUNT(&cpus);
1351 }
1352
1353 /* Clamp to the number of RSS buckets */
1354 queues = imin(queues, rss_getnumbuckets());
1355
1356 /*
1357 * Clamp the number of queue pairs to the minimum of the requested Tx
1358 * and Rx queues.
1359 */
1360 queues = imin(queues, sc->ifc_sysctl_ntxqs ?: scctx->isc_ntxqsets);
1361 queues = imin(queues, sc->ifc_sysctl_nrxqs ?: scctx->isc_nrxqsets);
1362
1363 /*
1364 * Determine the number of vectors to request. Note that we also need
1365 * to allocate one vector for administrative tasks.
1366 */
1367 requested = queues + 1;
1368
1369 vectors = requested;
1370
1371 err = pci_alloc_msix(dev, &vectors);
1372 if (err) {
1373 device_printf(dev, "Failed to allocate %d MSI-X vectors, err %s\n",
1374 vectors, ice_err_str(err));
1375 goto err_free_msix_table;
1376 }
1377
1378 /* If we don't receive enough vectors, reduce demands */
1379 if (vectors < requested) {
1380 int diff = requested - vectors;
1381
1382 device_printf(dev, "Requested %d MSI-X vectors, but got only %d\n",
1383 requested, vectors);
1384
1385 /*
1386 * If we still have a difference, we need to reduce the number
1387 * of queue pairs.
1388 *
1389 * However, we still need at least one vector for the admin
1390 * interrupt and one queue pair.
1391 */
1392 if (queues <= diff) {
1393 device_printf(dev, "Unable to allocate sufficient MSI-X vectors\n");
1394 err = (ERANGE);
1395 goto err_pci_release_msi;
1396 }
1397
1398 queues -= diff;
1399 }
1400
1401 device_printf(dev, "Using %d Tx and Rx queues\n", queues);
1402 device_printf(dev, "Using MSI-X interrupts with %d vectors\n",
1403 vectors);
1404
1405 scctx->isc_vectors = vectors;
1406 scctx->isc_nrxqsets = queues;
1407 scctx->isc_ntxqsets = queues;
1408 scctx->isc_intr = IFLIB_INTR_MSIX;
1409
1410 /* Interrupt allocation tracking isn't required in recovery mode,
1411 * since neither RDMA nor VFs are enabled.
1412 */
1413 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1414 return (0);
1415
1416 /* Keep track of which interrupt indices are being used for what */
1417 sc->lan_vectors = vectors;
1418 err = ice_resmgr_assign_contiguous(&sc->imgr, sc->pf_imap, sc->lan_vectors);
1419 if (err) {
1420 device_printf(dev, "Unable to assign PF interrupt mapping: %s\n",
1421 ice_err_str(err));
1422 goto err_pci_release_msi;
1423 }
1424
1425 return (0);
1426
1427 err_pci_release_msi:
1428 pci_release_msi(dev);
1429 err_free_msix_table:
1430 if (sc->msix_table != NULL) {
1431 bus_release_resource(sc->dev, SYS_RES_MEMORY,
1432 rman_get_rid(sc->msix_table),
1433 sc->msix_table);
1434 sc->msix_table = NULL;
1435 }
1436
1437 return (err);
1438 }
1439
1440 /**
1441 * ice_if_msix_intr_assign - Assign MSI-X interrupt vectors to queues
1442 * @ctx: the iflib context structure
1443 * @msix: the number of vectors we were assigned
1444 *
1445 * Called by iflib to assign MSI-X vectors to queues. Currently requires that
1446 * we get at least the same number of vectors as we have queues, and that we
1447 * always have the same number of Tx and Rx queues.
1448 *
1449 * Tx queues use a softirq instead of using their own hardware interrupt.
1450 */
1451 static int
ice_if_msix_intr_assign(if_ctx_t ctx,int msix)1452 ice_if_msix_intr_assign(if_ctx_t ctx, int msix)
1453 {
1454 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1455 struct ice_vsi *vsi = &sc->pf_vsi;
1456 int err, i, vector;
1457
1458 ASSERT_CTX_LOCKED(sc);
1459
1460 if (vsi->num_rx_queues != vsi->num_tx_queues) {
1461 device_printf(sc->dev,
1462 "iflib requested %d Tx queues, and %d Rx queues, but the driver isn't able to support a differing number of Tx and Rx queues\n",
1463 vsi->num_tx_queues, vsi->num_rx_queues);
1464 return (EOPNOTSUPP);
1465 }
1466
1467 if (msix < (vsi->num_rx_queues + 1)) {
1468 device_printf(sc->dev,
1469 "Not enough MSI-X vectors to assign one vector to each queue pair\n");
1470 return (EOPNOTSUPP);
1471 }
1472
1473 /* Save the number of vectors for future use */
1474 sc->num_irq_vectors = vsi->num_rx_queues + 1;
1475
1476 /* Allocate space to store the IRQ vector data */
1477 if (!(sc->irqvs =
1478 (struct ice_irq_vector *) malloc(sizeof(struct ice_irq_vector) * (sc->num_irq_vectors),
1479 M_ICE, M_NOWAIT))) {
1480 device_printf(sc->dev,
1481 "Unable to allocate irqv memory\n");
1482 return (ENOMEM);
1483 }
1484
1485 /* Administrative interrupt events will use vector 0 */
1486 err = iflib_irq_alloc_generic(ctx, &sc->irqvs[0].irq, 1, IFLIB_INTR_ADMIN,
1487 ice_msix_admin, sc, 0, "admin");
1488 if (err) {
1489 device_printf(sc->dev,
1490 "Failed to register Admin queue handler: %s\n",
1491 ice_err_str(err));
1492 goto free_irqvs;
1493 }
1494 sc->irqvs[0].me = 0;
1495
1496 /* Do not allocate queue interrupts when in recovery mode */
1497 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1498 return (0);
1499
1500 for (i = 0, vector = 1; i < vsi->num_rx_queues; i++, vector++) {
1501 struct ice_rx_queue *rxq = &vsi->rx_queues[i];
1502 struct ice_tx_queue *txq = &vsi->tx_queues[i];
1503 int rid = vector + 1;
1504 char irq_name[16];
1505
1506 snprintf(irq_name, sizeof(irq_name), "rxq%d", i);
1507 err = iflib_irq_alloc_generic(ctx, &sc->irqvs[vector].irq, rid,
1508 IFLIB_INTR_RXTX, ice_msix_que,
1509 rxq, rxq->me, irq_name);
1510 if (err) {
1511 device_printf(sc->dev,
1512 "Failed to allocate q int %d err: %s\n",
1513 i, ice_err_str(err));
1514 vector--;
1515 i--;
1516 goto fail;
1517 }
1518 sc->irqvs[vector].me = vector;
1519 rxq->irqv = &sc->irqvs[vector];
1520
1521 bzero(irq_name, sizeof(irq_name));
1522
1523 snprintf(irq_name, sizeof(irq_name), "txq%d", i);
1524 iflib_softirq_alloc_generic(ctx, &sc->irqvs[vector].irq,
1525 IFLIB_INTR_TX, txq,
1526 txq->me, irq_name);
1527 txq->irqv = &sc->irqvs[vector];
1528 }
1529
1530 return (0);
1531 fail:
1532 for (; i >= 0; i--, vector--)
1533 iflib_irq_free(ctx, &sc->irqvs[vector].irq);
1534 iflib_irq_free(ctx, &sc->irqvs[0].irq);
1535 free_irqvs:
1536 free(sc->irqvs, M_ICE);
1537 sc->irqvs = NULL;
1538 return err;
1539 }
1540
1541 /**
1542 * ice_if_mtu_set - Set the device MTU
1543 * @ctx: iflib context structure
1544 * @mtu: the MTU requested
1545 *
1546 * Called by iflib to configure the device's Maximum Transmission Unit (MTU).
1547 *
1548 * @pre assumes the caller holds the iflib CTX lock
1549 */
1550 static int
ice_if_mtu_set(if_ctx_t ctx,uint32_t mtu)1551 ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
1552 {
1553 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1554
1555 ASSERT_CTX_LOCKED(sc);
1556
1557 /* Do not support configuration when in recovery mode */
1558 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1559 return (ENOSYS);
1560
1561 if (mtu < ICE_MIN_MTU || mtu > ICE_MAX_MTU)
1562 return (EINVAL);
1563
1564 sc->scctx->isc_max_frame_size = mtu +
1565 ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN;
1566
1567 sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size;
1568
1569 return (0);
1570 }
1571
1572 /**
1573 * ice_if_intr_enable - Enable device interrupts
1574 * @ctx: iflib context structure
1575 *
1576 * Called by iflib to request enabling device interrupts.
1577 */
1578 static void
ice_if_intr_enable(if_ctx_t ctx)1579 ice_if_intr_enable(if_ctx_t ctx)
1580 {
1581 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1582 struct ice_vsi *vsi = &sc->pf_vsi;
1583 struct ice_hw *hw = &sc->hw;
1584
1585 ASSERT_CTX_LOCKED(sc);
1586
1587 /* Enable ITR 0 */
1588 ice_enable_intr(hw, sc->irqvs[0].me);
1589
1590 /* Do not enable queue interrupts in recovery mode */
1591 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1592 return;
1593
1594 /* Enable all queue interrupts */
1595 for (int i = 0; i < vsi->num_rx_queues; i++)
1596 ice_enable_intr(hw, vsi->rx_queues[i].irqv->me);
1597 }
1598
1599 /**
1600 * ice_if_intr_disable - Disable device interrupts
1601 * @ctx: iflib context structure
1602 *
1603 * Called by iflib to request disabling device interrupts.
1604 */
1605 static void
ice_if_intr_disable(if_ctx_t ctx)1606 ice_if_intr_disable(if_ctx_t ctx)
1607 {
1608 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1609 struct ice_hw *hw = &sc->hw;
1610 unsigned int i;
1611
1612 ASSERT_CTX_LOCKED(sc);
1613
1614 /* IFDI_INTR_DISABLE may be called prior to interrupts actually being
1615 * assigned to queues. Instead of assuming that the interrupt
1616 * assignment in the rx_queues structure is valid, just disable all
1617 * possible interrupts
1618 *
1619 * Note that we choose not to disable ITR 0 because this handles the
1620 * AdminQ interrupts, and we want to keep processing these even when
1621 * the interface is offline.
1622 */
1623 for (i = 1; i < hw->func_caps.common_cap.num_msix_vectors; i++)
1624 ice_disable_intr(hw, i);
1625 }
1626
1627 /**
1628 * ice_if_rx_queue_intr_enable - Enable a specific Rx queue interrupt
1629 * @ctx: iflib context structure
1630 * @rxqid: the Rx queue to enable
1631 *
1632 * Enable a specific Rx queue interrupt.
1633 *
1634 * This function is not protected by the iflib CTX lock.
1635 */
1636 static int
ice_if_rx_queue_intr_enable(if_ctx_t ctx,uint16_t rxqid)1637 ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
1638 {
1639 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1640 struct ice_vsi *vsi = &sc->pf_vsi;
1641 struct ice_hw *hw = &sc->hw;
1642
1643 /* Do not enable queue interrupts in recovery mode */
1644 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1645 return (ENOSYS);
1646
1647 ice_enable_intr(hw, vsi->rx_queues[rxqid].irqv->me);
1648 return (0);
1649 }
1650
1651 /**
1652 * ice_if_tx_queue_intr_enable - Enable a specific Tx queue interrupt
1653 * @ctx: iflib context structure
1654 * @txqid: the Tx queue to enable
1655 *
1656 * Enable a specific Tx queue interrupt.
1657 *
1658 * This function is not protected by the iflib CTX lock.
1659 */
1660 static int
ice_if_tx_queue_intr_enable(if_ctx_t ctx,uint16_t txqid)1661 ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid)
1662 {
1663 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1664 struct ice_vsi *vsi = &sc->pf_vsi;
1665 struct ice_hw *hw = &sc->hw;
1666
1667 /* Do not enable queue interrupts in recovery mode */
1668 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1669 return (ENOSYS);
1670
1671 ice_enable_intr(hw, vsi->tx_queues[txqid].irqv->me);
1672 return (0);
1673 }
1674
1675 /**
1676 * ice_if_promisc_set - Set device promiscuous mode
1677 * @ctx: iflib context structure
1678 * @flags: promiscuous flags to configure
1679 *
1680 * Called by iflib to configure device promiscuous mode.
1681 *
1682 * @remark Calls to this function will always overwrite the previous setting
1683 */
1684 static int
ice_if_promisc_set(if_ctx_t ctx,int flags)1685 ice_if_promisc_set(if_ctx_t ctx, int flags)
1686 {
1687 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1688 struct ice_hw *hw = &sc->hw;
1689 device_t dev = sc->dev;
1690 enum ice_status status;
1691 bool promisc_enable = flags & IFF_PROMISC;
1692 bool multi_enable = flags & IFF_ALLMULTI;
1693
1694 /* Do not support configuration when in recovery mode */
1695 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1696 return (ENOSYS);
1697
1698 if (multi_enable)
1699 return (EOPNOTSUPP);
1700
1701 if (promisc_enable) {
1702 status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx,
1703 ICE_VSI_PROMISC_MASK, 0);
1704 if (status && status != ICE_ERR_ALREADY_EXISTS) {
1705 device_printf(dev,
1706 "Failed to enable promiscuous mode for PF VSI, err %s aq_err %s\n",
1707 ice_status_str(status),
1708 ice_aq_str(hw->adminq.sq_last_status));
1709 return (EIO);
1710 }
1711 } else {
1712 status = ice_clear_vsi_promisc(hw, sc->pf_vsi.idx,
1713 ICE_VSI_PROMISC_MASK, 0);
1714 if (status) {
1715 device_printf(dev,
1716 "Failed to disable promiscuous mode for PF VSI, err %s aq_err %s\n",
1717 ice_status_str(status),
1718 ice_aq_str(hw->adminq.sq_last_status));
1719 return (EIO);
1720 }
1721 }
1722
1723 return (0);
1724 }
1725
1726 /**
1727 * ice_if_media_change - Change device media
1728 * @ctx: device ctx structure
1729 *
1730 * Called by iflib when a media change is requested. This operation is not
1731 * supported by the hardware, so we just return an error code.
1732 */
1733 static int
ice_if_media_change(if_ctx_t ctx)1734 ice_if_media_change(if_ctx_t ctx)
1735 {
1736 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1737
1738 device_printf(sc->dev, "Media change is not supported.\n");
1739 return (ENODEV);
1740 }
1741
1742 /**
1743 * ice_if_media_status - Report current device media
1744 * @ctx: iflib context structure
1745 * @ifmr: ifmedia request structure to update
1746 *
1747 * Updates the provided ifmr with current device media status, including link
1748 * status and media type.
1749 */
1750 static void
ice_if_media_status(if_ctx_t ctx,struct ifmediareq * ifmr)1751 ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr)
1752 {
1753 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1754 struct ice_link_status *li = &sc->hw.port_info->phy.link_info;
1755
1756 ifmr->ifm_status = IFM_AVALID;
1757 ifmr->ifm_active = IFM_ETHER;
1758
1759 /* Never report link up or media types when in recovery mode */
1760 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1761 return;
1762
1763 if (!sc->link_up)
1764 return;
1765
1766 ifmr->ifm_status |= IFM_ACTIVE;
1767 ifmr->ifm_active |= IFM_FDX;
1768
1769 if (li->phy_type_low)
1770 ifmr->ifm_active |= ice_get_phy_type_low(li->phy_type_low);
1771 else if (li->phy_type_high)
1772 ifmr->ifm_active |= ice_get_phy_type_high(li->phy_type_high);
1773 else
1774 ifmr->ifm_active |= IFM_UNKNOWN;
1775
1776 /* Report flow control status as well */
1777 if (li->an_info & ICE_AQ_LINK_PAUSE_TX)
1778 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1779 if (li->an_info & ICE_AQ_LINK_PAUSE_RX)
1780 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1781 }
1782
1783 /**
1784 * ice_init_tx_tracking - Initialize Tx queue software tracking values
1785 * @vsi: the VSI to initialize
1786 *
1787 * Initialize Tx queue software tracking values, including the Report Status
1788 * queue, and related software tracking values.
1789 */
1790 static void
ice_init_tx_tracking(struct ice_vsi * vsi)1791 ice_init_tx_tracking(struct ice_vsi *vsi)
1792 {
1793 struct ice_tx_queue *txq;
1794 size_t j;
1795 int i;
1796
1797 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) {
1798
1799 txq->tx_rs_cidx = txq->tx_rs_pidx = 0;
1800
1801 /* Initialize the last processed descriptor to be the end of
1802 * the ring, rather than the start, so that we avoid an
1803 * off-by-one error in ice_ift_txd_credits_update for the
1804 * first packet.
1805 */
1806 txq->tx_cidx_processed = txq->desc_count - 1;
1807
1808 for (j = 0; j < txq->desc_count; j++)
1809 txq->tx_rsq[j] = QIDX_INVALID;
1810 }
1811 }
1812
1813 /**
1814 * ice_update_rx_mbuf_sz - Update the Rx buffer size for all queues
1815 * @sc: the device softc
1816 *
1817 * Called to update the Rx queue mbuf_sz parameter for configuring the receive
1818 * buffer sizes when programming hardware.
1819 */
1820 static void
ice_update_rx_mbuf_sz(struct ice_softc * sc)1821 ice_update_rx_mbuf_sz(struct ice_softc *sc)
1822 {
1823 uint32_t mbuf_sz = iflib_get_rx_mbuf_sz(sc->ctx);
1824 struct ice_vsi *vsi = &sc->pf_vsi;
1825
1826 MPASS(mbuf_sz <= UINT16_MAX);
1827 vsi->mbuf_sz = mbuf_sz;
1828 }
1829
1830 /**
1831 * ice_if_init - Initialize the device
1832 * @ctx: iflib ctx structure
1833 *
1834 * Called by iflib to bring the device up, i.e. ifconfig ice0 up. Initializes
1835 * device filters and prepares the Tx and Rx engines.
1836 *
1837 * @pre assumes the caller holds the iflib CTX lock
1838 */
1839 static void
ice_if_init(if_ctx_t ctx)1840 ice_if_init(if_ctx_t ctx)
1841 {
1842 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1843 device_t dev = sc->dev;
1844 int err;
1845
1846 ASSERT_CTX_LOCKED(sc);
1847
1848 /*
1849 * We've seen an issue with 11.3/12.1 where sideband routines are
1850 * called after detach is called. This would call routines after
1851 * if_stop, causing issues with the teardown process. This has
1852 * seemingly been fixed in STABLE snapshots, but it seems like a
1853 * good idea to have this guard here regardless.
1854 */
1855 if (ice_driver_is_detaching(sc))
1856 return;
1857
1858 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1859 return;
1860
1861 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) {
1862 device_printf(sc->dev, "request to start interface cannot be completed as the device failed to reset\n");
1863 return;
1864 }
1865
1866 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
1867 device_printf(sc->dev, "request to start interface while device is prepared for impending reset\n");
1868 return;
1869 }
1870
1871 ice_update_rx_mbuf_sz(sc);
1872
1873 /* Update the MAC address... User might use a LAA */
1874 err = ice_update_laa_mac(sc);
1875 if (err) {
1876 device_printf(dev,
1877 "LAA address change failed, err %s\n",
1878 ice_err_str(err));
1879 return;
1880 }
1881
1882 /* Initialize software Tx tracking values */
1883 ice_init_tx_tracking(&sc->pf_vsi);
1884
1885 err = ice_cfg_vsi_for_tx(&sc->pf_vsi);
1886 if (err) {
1887 device_printf(dev,
1888 "Unable to configure the main VSI for Tx: %s\n",
1889 ice_err_str(err));
1890 return;
1891 }
1892
1893 err = ice_cfg_vsi_for_rx(&sc->pf_vsi);
1894 if (err) {
1895 device_printf(dev,
1896 "Unable to configure the main VSI for Rx: %s\n",
1897 ice_err_str(err));
1898 goto err_cleanup_tx;
1899 }
1900
1901 err = ice_control_rx_queues(&sc->pf_vsi, true);
1902 if (err) {
1903 device_printf(dev,
1904 "Unable to enable Rx rings for transmit: %s\n",
1905 ice_err_str(err));
1906 goto err_cleanup_tx;
1907 }
1908
1909 err = ice_cfg_pf_default_mac_filters(sc);
1910 if (err) {
1911 device_printf(dev,
1912 "Unable to configure default MAC filters: %s\n",
1913 ice_err_str(err));
1914 goto err_stop_rx;
1915 }
1916
1917 /* We use software interrupts for Tx, so we only program the hardware
1918 * interrupts for Rx.
1919 */
1920 ice_configure_rxq_interrupts(&sc->pf_vsi);
1921 ice_configure_rx_itr(&sc->pf_vsi);
1922
1923 /* Configure promiscuous mode */
1924 ice_if_promisc_set(ctx, if_getflags(sc->ifp));
1925
1926 ice_set_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED);
1927 return;
1928
1929 err_stop_rx:
1930 ice_control_rx_queues(&sc->pf_vsi, false);
1931 err_cleanup_tx:
1932 ice_vsi_disable_tx(&sc->pf_vsi);
1933 }
1934
1935 /**
1936 * ice_poll_for_media_avail - Re-enable link if media is detected
1937 * @sc: device private structure
1938 *
1939 * Intended to be called from the driver's timer function, this function
1940 * sends the Get Link Status AQ command and re-enables HW link if the
1941 * command says that media is available.
1942 *
1943 * If the driver doesn't have the "NO_MEDIA" state set, then this does nothing,
1944 * since media removal events are supposed to be sent to the driver through
1945 * a link status event.
1946 */
1947 static void
ice_poll_for_media_avail(struct ice_softc * sc)1948 ice_poll_for_media_avail(struct ice_softc *sc)
1949 {
1950 struct ice_hw *hw = &sc->hw;
1951 struct ice_port_info *pi = hw->port_info;
1952
1953 if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) {
1954 pi->phy.get_link_info = true;
1955 ice_get_link_status(pi, &sc->link_up);
1956
1957 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
1958 enum ice_status status;
1959
1960 /* Re-enable link and re-apply user link settings */
1961 ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS_FEC_FC);
1962
1963 /* Update the OS about changes in media capability */
1964 status = ice_add_media_types(sc, sc->media);
1965 if (status)
1966 device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n",
1967 ice_status_str(status),
1968 ice_aq_str(hw->adminq.sq_last_status));
1969
1970 ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA);
1971 }
1972 }
1973 }
1974
1975 /**
1976 * ice_if_timer - called by iflib periodically
1977 * @ctx: iflib ctx structure
1978 * @qid: the queue this timer was called for
1979 *
1980 * This callback is triggered by iflib periodically. We use it to update the
1981 * hw statistics.
1982 *
1983 * @remark this function is not protected by the iflib CTX lock.
1984 */
1985 static void
ice_if_timer(if_ctx_t ctx,uint16_t qid)1986 ice_if_timer(if_ctx_t ctx, uint16_t qid)
1987 {
1988 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1989 uint64_t prev_link_xoff_rx = sc->stats.cur.link_xoff_rx;
1990
1991 if (qid != 0)
1992 return;
1993
1994 /* Do not attempt to update stats when in recovery mode */
1995 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1996 return;
1997
1998 /* Update device statistics */
1999 ice_update_pf_stats(sc);
2000
2001 /*
2002 * For proper watchdog management, the iflib stack needs to know if
2003 * we've been paused during the last interval. Check if the
2004 * link_xoff_rx stat changed, and set the isc_pause_frames, if so.
2005 */
2006 if (sc->stats.cur.link_xoff_rx != prev_link_xoff_rx)
2007 sc->scctx->isc_pause_frames = 1;
2008
2009 /* Update the primary VSI stats */
2010 ice_update_vsi_hw_stats(&sc->pf_vsi);
2011 }
2012
2013 /**
2014 * ice_admin_timer - called periodically to trigger the admin task
2015 * @arg: callout(9) argument pointing to the device private softc structure
2016 *
2017 * Timer function used as part of a callout(9) timer that will periodically
2018 * trigger the admin task, even when the interface is down.
2019 *
2020 * @remark this function is not called by iflib and is not protected by the
2021 * iflib CTX lock.
2022 *
2023 * @remark because this is a callout function, it cannot sleep and should not
2024 * attempt taking the iflib CTX lock.
2025 */
2026 static void
ice_admin_timer(void * arg)2027 ice_admin_timer(void *arg)
2028 {
2029 struct ice_softc *sc = (struct ice_softc *)arg;
2030
2031 /*
2032 * There is a point where callout routines are no longer
2033 * cancelable. So there exists a window of time where the
2034 * driver enters detach() and tries to cancel the callout, but the
2035 * callout routine has passed the cancellation point. The detach()
2036 * routine is unaware of this and tries to free resources that the
2037 * callout routine needs. So we check for the detach state flag to
2038 * at least shrink the window of opportunity.
2039 */
2040 if (ice_driver_is_detaching(sc))
2041 return;
2042
2043 /* Fire off the admin task */
2044 iflib_admin_intr_deferred(sc->ctx);
2045
2046 /* Reschedule the admin timer */
2047 callout_schedule(&sc->admin_timer, hz/2);
2048 }
2049
2050 /**
2051 * ice_transition_recovery_mode - Transition to recovery mode
2052 * @sc: the device private softc
2053 *
2054 * Called when the driver detects that the firmware has entered recovery mode
2055 * at run time.
2056 */
2057 static void
ice_transition_recovery_mode(struct ice_softc * sc)2058 ice_transition_recovery_mode(struct ice_softc *sc)
2059 {
2060 struct ice_vsi *vsi = &sc->pf_vsi;
2061 int i;
2062
2063 device_printf(sc->dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
2064
2065 /* Tell the stack that the link has gone down */
2066 iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0);
2067
2068 /* Request that the device be re-initialized */
2069 ice_request_stack_reinit(sc);
2070
2071 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en);
2072 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2073
2074 ice_vsi_del_txqs_ctx(vsi);
2075 ice_vsi_del_rxqs_ctx(vsi);
2076
2077 for (i = 0; i < sc->num_available_vsi; i++) {
2078 if (sc->all_vsi[i])
2079 ice_release_vsi(sc->all_vsi[i]);
2080 }
2081 sc->num_available_vsi = 0;
2082
2083 if (sc->all_vsi) {
2084 free(sc->all_vsi, M_ICE);
2085 sc->all_vsi = NULL;
2086 }
2087
2088 /* Destroy the interrupt manager */
2089 ice_resmgr_destroy(&sc->imgr);
2090 /* Destroy the queue managers */
2091 ice_resmgr_destroy(&sc->tx_qmgr);
2092 ice_resmgr_destroy(&sc->rx_qmgr);
2093
2094 ice_deinit_hw(&sc->hw);
2095 }
2096
2097 /**
2098 * ice_transition_safe_mode - Transition to safe mode
2099 * @sc: the device private softc
2100 *
2101 * Called when the driver attempts to reload the DDP package during a device
2102 * reset, and the new download fails. If so, we must transition to safe mode
2103 * at run time.
2104 *
2105 * @remark although safe mode normally allocates only a single queue, we can't
2106 * change the number of queues dynamically when using iflib. Due to this, we
2107 * do not attempt to reduce the number of queues.
2108 */
2109 static void
ice_transition_safe_mode(struct ice_softc * sc)2110 ice_transition_safe_mode(struct ice_softc *sc)
2111 {
2112 /* Indicate that we are in Safe mode */
2113 ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_cap);
2114 ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_en);
2115
2116 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en);
2117 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2118
2119 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap);
2120 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_en);
2121 }
2122
2123 /**
2124 * ice_if_update_admin_status - update admin status
2125 * @ctx: iflib ctx structure
2126 *
2127 * Called by iflib to update the admin status. For our purposes, this means
2128 * check the adminq, and update the link status. It's ultimately triggered by
2129 * our admin interrupt, or by the ice_if_timer periodically.
2130 *
2131 * @pre assumes the caller holds the iflib CTX lock
2132 */
2133 static void
ice_if_update_admin_status(if_ctx_t ctx)2134 ice_if_update_admin_status(if_ctx_t ctx)
2135 {
2136 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2137 enum ice_fw_modes fw_mode;
2138 bool reschedule = false;
2139 u16 pending = 0;
2140
2141 ASSERT_CTX_LOCKED(sc);
2142
2143 /* Check if the firmware entered recovery mode at run time */
2144 fw_mode = ice_get_fw_mode(&sc->hw);
2145 if (fw_mode == ICE_FW_MODE_REC) {
2146 if (!ice_testandset_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2147 /* If we just entered recovery mode, log a warning to
2148 * the system administrator and deinit driver state
2149 * that is no longer functional.
2150 */
2151 ice_transition_recovery_mode(sc);
2152 }
2153 } else if (fw_mode == ICE_FW_MODE_ROLLBACK) {
2154 if (!ice_testandset_state(&sc->state, ICE_STATE_ROLLBACK_MODE)) {
2155 /* Rollback mode isn't fatal, but we don't want to
2156 * repeatedly post a message about it.
2157 */
2158 ice_print_rollback_msg(&sc->hw);
2159 }
2160 }
2161
2162 /* Handle global reset events */
2163 ice_handle_reset_event(sc);
2164
2165 /* Handle PF reset requests */
2166 ice_handle_pf_reset_request(sc);
2167
2168 /* Handle MDD events */
2169 ice_handle_mdd_event(sc);
2170
2171 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED) ||
2172 ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET) ||
2173 ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2174 /*
2175 * If we know the control queues are disabled, skip processing
2176 * the control queues entirely.
2177 */
2178 ;
2179 } else if (ice_testandclear_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING)) {
2180 ice_process_ctrlq(sc, ICE_CTL_Q_ADMIN, &pending);
2181 if (pending > 0)
2182 reschedule = true;
2183
2184 ice_process_ctrlq(sc, ICE_CTL_Q_MAILBOX, &pending);
2185 if (pending > 0)
2186 reschedule = true;
2187 }
2188
2189 /* Poll for link up */
2190 ice_poll_for_media_avail(sc);
2191
2192 /* Check and update link status */
2193 ice_update_link_status(sc, false);
2194
2195 /*
2196 * If there are still messages to process, we need to reschedule
2197 * ourselves. Otherwise, we can just re-enable the interrupt. We'll be
2198 * woken up at the next interrupt or timer event.
2199 */
2200 if (reschedule) {
2201 ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);
2202 iflib_admin_intr_deferred(ctx);
2203 } else {
2204 ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2205 }
2206 }
2207
2208 /**
2209 * ice_prepare_for_reset - Prepare device for an impending reset
2210 * @sc: The device private softc
2211 *
2212 * Prepare the driver for an impending reset, shutting down VSIs, clearing the
2213 * scheduler setup, and shutting down controlqs. Uses the
2214 * ICE_STATE_PREPARED_FOR_RESET to indicate whether we've already prepared the
2215 * driver for reset or not.
2216 */
2217 static void
ice_prepare_for_reset(struct ice_softc * sc)2218 ice_prepare_for_reset(struct ice_softc *sc)
2219 {
2220 struct ice_hw *hw = &sc->hw;
2221
2222 /* If we're already prepared, there's nothing to do */
2223 if (ice_testandset_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET))
2224 return;
2225
2226 log(LOG_INFO, "%s: preparing to reset device logic\n", sc->ifp->if_xname);
2227
2228 /* In recovery mode, hardware is not initialized */
2229 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2230 return;
2231
2232 /* Release the main PF VSI queue mappings */
2233 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2234 sc->pf_vsi.num_tx_queues);
2235 ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap,
2236 sc->pf_vsi.num_rx_queues);
2237
2238 ice_clear_hw_tbls(hw);
2239
2240 if (hw->port_info)
2241 ice_sched_clear_port(hw->port_info);
2242
2243 ice_shutdown_all_ctrlq(hw);
2244 }
2245
2246 /**
2247 * ice_rebuild_pf_vsi_qmap - Rebuild the main PF VSI queue mapping
2248 * @sc: the device softc pointer
2249 *
2250 * Loops over the Tx and Rx queues for the main PF VSI and reassigns the queue
2251 * mapping after a reset occurred.
2252 */
2253 static int
ice_rebuild_pf_vsi_qmap(struct ice_softc * sc)2254 ice_rebuild_pf_vsi_qmap(struct ice_softc *sc)
2255 {
2256 struct ice_vsi *vsi = &sc->pf_vsi;
2257 struct ice_tx_queue *txq;
2258 struct ice_rx_queue *rxq;
2259 int err, i;
2260
2261 /* Re-assign Tx queues from PF space to the main VSI */
2262 err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap,
2263 vsi->num_tx_queues);
2264 if (err) {
2265 device_printf(sc->dev, "Unable to re-assign PF Tx queues: %s\n",
2266 ice_err_str(err));
2267 return (err);
2268 }
2269
2270 /* Re-assign Rx queues from PF space to this VSI */
2271 err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap,
2272 vsi->num_rx_queues);
2273 if (err) {
2274 device_printf(sc->dev, "Unable to re-assign PF Rx queues: %s\n",
2275 ice_err_str(err));
2276 goto err_release_tx_queues;
2277 }
2278
2279 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
2280
2281 /* Re-assign Tx queue tail pointers */
2282 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++)
2283 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]);
2284
2285 /* Re-assign Rx queue tail pointers */
2286 for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++)
2287 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]);
2288
2289 return (0);
2290
2291 err_release_tx_queues:
2292 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2293 sc->pf_vsi.num_tx_queues);
2294
2295 return (err);
2296 }
2297
2298 /* determine if the iflib context is active */
2299 #define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING))
2300
2301 /**
2302 * ice_rebuild_recovery_mode - Rebuild driver state while in recovery mode
2303 * @sc: The device private softc
2304 *
2305 * Handle a driver rebuild while in recovery mode. This will only rebuild the
2306 * limited functionality supported while in recovery mode.
2307 */
2308 static void
ice_rebuild_recovery_mode(struct ice_softc * sc)2309 ice_rebuild_recovery_mode(struct ice_softc *sc)
2310 {
2311 device_t dev = sc->dev;
2312
2313 /* enable PCIe bus master */
2314 pci_enable_busmaster(dev);
2315
2316 /* Configure interrupt causes for the administrative interrupt */
2317 ice_configure_misc_interrupts(sc);
2318
2319 /* Enable ITR 0 right away, so that we can handle admin interrupts */
2320 ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2321
2322 /* Now that the rebuild is finished, we're no longer prepared to reset */
2323 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
2324
2325 log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname);
2326
2327 /* In order to completely restore device functionality, the iflib core
2328 * needs to be reset. We need to request an iflib reset. Additionally,
2329 * because the state of IFC_DO_RESET is cached within task_fn_admin in
2330 * the iflib core, we also want re-run the admin task so that iflib
2331 * resets immediately instead of waiting for the next interrupt.
2332 */
2333 ice_request_stack_reinit(sc);
2334
2335 return;
2336 }
2337
2338 /**
2339 * ice_rebuild - Rebuild driver state post reset
2340 * @sc: The device private softc
2341 *
2342 * Restore driver state after a reset occurred. Restart the controlqs, setup
2343 * the hardware port, and re-enable the VSIs.
2344 */
2345 static void
ice_rebuild(struct ice_softc * sc)2346 ice_rebuild(struct ice_softc *sc)
2347 {
2348 struct ice_hw *hw = &sc->hw;
2349 device_t dev = sc->dev;
2350 enum ice_status status;
2351 int err;
2352
2353 sc->rebuild_ticks = ticks;
2354
2355 /* If we're rebuilding, then a reset has succeeded. */
2356 ice_clear_state(&sc->state, ICE_STATE_RESET_FAILED);
2357
2358 /*
2359 * If the firmware is in recovery mode, only restore the limited
2360 * functionality supported by recovery mode.
2361 */
2362 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2363 ice_rebuild_recovery_mode(sc);
2364 return;
2365 }
2366
2367 /* enable PCIe bus master */
2368 pci_enable_busmaster(dev);
2369
2370 status = ice_init_all_ctrlq(hw);
2371 if (status) {
2372 device_printf(dev, "failed to re-init controlqs, err %s\n",
2373 ice_status_str(status));
2374 goto err_shutdown_ctrlq;
2375 }
2376
2377 /* Query the allocated resources for Tx scheduler */
2378 status = ice_sched_query_res_alloc(hw);
2379 if (status) {
2380 device_printf(dev,
2381 "Failed to query scheduler resources, err %s aq_err %s\n",
2382 ice_status_str(status),
2383 ice_aq_str(hw->adminq.sq_last_status));
2384 goto err_shutdown_ctrlq;
2385 }
2386
2387 /* Re-enable FW logging. Keep going even if this fails */
2388 status = ice_fwlog_set(hw, &hw->fwlog_cfg);
2389 if (!status) {
2390 /*
2391 * We should have the most updated cached copy of the
2392 * configuration, regardless of whether we're rebuilding
2393 * or not. So we'll simply check to see if logging was
2394 * enabled pre-rebuild.
2395 */
2396 if (hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) {
2397 status = ice_fwlog_register(hw);
2398 if (status)
2399 device_printf(dev, "failed to re-register fw logging, err %s aq_err %s\n",
2400 ice_status_str(status),
2401 ice_aq_str(hw->adminq.sq_last_status));
2402 }
2403 } else
2404 device_printf(dev, "failed to rebuild fw logging configuration, err %s aq_err %s\n",
2405 ice_status_str(status),
2406 ice_aq_str(hw->adminq.sq_last_status));
2407
2408 err = ice_send_version(sc);
2409 if (err)
2410 goto err_shutdown_ctrlq;
2411
2412 err = ice_init_link_events(sc);
2413 if (err) {
2414 device_printf(dev, "ice_init_link_events failed: %s\n",
2415 ice_err_str(err));
2416 goto err_shutdown_ctrlq;
2417 }
2418
2419 status = ice_clear_pf_cfg(hw);
2420 if (status) {
2421 device_printf(dev, "failed to clear PF configuration, err %s\n",
2422 ice_status_str(status));
2423 goto err_shutdown_ctrlq;
2424 }
2425
2426 ice_clear_pxe_mode(hw);
2427
2428 status = ice_get_caps(hw);
2429 if (status) {
2430 device_printf(dev, "failed to get capabilities, err %s\n",
2431 ice_status_str(status));
2432 goto err_shutdown_ctrlq;
2433 }
2434
2435 status = ice_sched_init_port(hw->port_info);
2436 if (status) {
2437 device_printf(dev, "failed to initialize port, err %s\n",
2438 ice_status_str(status));
2439 goto err_sched_cleanup;
2440 }
2441
2442 /* If we previously loaded the package, it needs to be reloaded now */
2443 if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) {
2444 status = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size);
2445 if (status) {
2446 ice_log_pkg_init(sc, &status);
2447
2448 ice_transition_safe_mode(sc);
2449 }
2450 }
2451
2452 ice_reset_pf_stats(sc);
2453
2454 err = ice_rebuild_pf_vsi_qmap(sc);
2455 if (err) {
2456 device_printf(sc->dev, "Unable to re-assign main VSI queues, err %s\n",
2457 ice_err_str(err));
2458 goto err_sched_cleanup;
2459 }
2460 err = ice_initialize_vsi(&sc->pf_vsi);
2461 if (err) {
2462 device_printf(sc->dev, "Unable to re-initialize Main VSI, err %s\n",
2463 ice_err_str(err));
2464 goto err_release_queue_allocations;
2465 }
2466
2467 /* Replay all VSI configuration */
2468 err = ice_replay_all_vsi_cfg(sc);
2469 if (err)
2470 goto err_deinit_pf_vsi;
2471
2472 /* Re-enable FW health event reporting */
2473 ice_init_health_events(sc);
2474
2475 /* Reconfigure the main PF VSI for RSS */
2476 err = ice_config_rss(&sc->pf_vsi);
2477 if (err) {
2478 device_printf(sc->dev,
2479 "Unable to reconfigure RSS for the main VSI, err %s\n",
2480 ice_err_str(err));
2481 goto err_deinit_pf_vsi;
2482 }
2483
2484 /* Refresh link status */
2485 ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED);
2486 sc->hw.port_info->phy.get_link_info = true;
2487 ice_get_link_status(sc->hw.port_info, &sc->link_up);
2488 ice_update_link_status(sc, true);
2489
2490 /* Configure interrupt causes for the administrative interrupt */
2491 ice_configure_misc_interrupts(sc);
2492
2493 /* Enable ITR 0 right away, so that we can handle admin interrupts */
2494 ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2495
2496 /* Now that the rebuild is finished, we're no longer prepared to reset */
2497 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
2498
2499 log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname);
2500
2501 /* In order to completely restore device functionality, the iflib core
2502 * needs to be reset. We need to request an iflib reset. Additionally,
2503 * because the state of IFC_DO_RESET is cached within task_fn_admin in
2504 * the iflib core, we also want re-run the admin task so that iflib
2505 * resets immediately instead of waiting for the next interrupt.
2506 */
2507 ice_request_stack_reinit(sc);
2508
2509 return;
2510
2511 err_deinit_pf_vsi:
2512 ice_deinit_vsi(&sc->pf_vsi);
2513 err_release_queue_allocations:
2514 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2515 sc->pf_vsi.num_tx_queues);
2516 ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap,
2517 sc->pf_vsi.num_rx_queues);
2518 err_sched_cleanup:
2519 ice_sched_cleanup_all(hw);
2520 err_shutdown_ctrlq:
2521 ice_shutdown_all_ctrlq(hw);
2522 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2523 device_printf(dev, "Driver rebuild failed, please reload the device driver\n");
2524 }
2525
2526 /**
2527 * ice_handle_reset_event - Handle reset events triggered by OICR
2528 * @sc: The device private softc
2529 *
2530 * Handle reset events triggered by an OICR notification. This includes CORER,
2531 * GLOBR, and EMPR resets triggered by software on this or any other PF or by
2532 * firmware.
2533 *
2534 * @pre assumes the iflib context lock is held, and will unlock it while
2535 * waiting for the hardware to finish reset.
2536 */
2537 static void
ice_handle_reset_event(struct ice_softc * sc)2538 ice_handle_reset_event(struct ice_softc *sc)
2539 {
2540 struct ice_hw *hw = &sc->hw;
2541 enum ice_status status;
2542 device_t dev = sc->dev;
2543
2544 /* When a CORER, GLOBR, or EMPR is about to happen, the hardware will
2545 * trigger an OICR interrupt. Our OICR handler will determine when
2546 * this occurs and set the ICE_STATE_RESET_OICR_RECV bit as
2547 * appropriate.
2548 */
2549 if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_OICR_RECV))
2550 return;
2551
2552 ice_prepare_for_reset(sc);
2553
2554 /*
2555 * Release the iflib context lock and wait for the device to finish
2556 * resetting.
2557 */
2558 IFLIB_CTX_UNLOCK(sc);
2559 status = ice_check_reset(hw);
2560 IFLIB_CTX_LOCK(sc);
2561 if (status) {
2562 device_printf(dev, "Device never came out of reset, err %s\n",
2563 ice_status_str(status));
2564 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2565 return;
2566 }
2567
2568 /* We're done with the reset, so we can rebuild driver state */
2569 sc->hw.reset_ongoing = false;
2570 ice_rebuild(sc);
2571
2572 /* In the unlikely event that a PF reset request occurs at the same
2573 * time as a global reset, clear the request now. This avoids
2574 * resetting a second time right after we reset due to a global event.
2575 */
2576 if (ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ))
2577 device_printf(dev, "Ignoring PFR request that occurred while a reset was ongoing\n");
2578 }
2579
2580 /**
2581 * ice_handle_pf_reset_request - Initiate PF reset requested by software
2582 * @sc: The device private softc
2583 *
2584 * Initiate a PF reset requested by software. We handle this in the admin task
2585 * so that only one thread actually handles driver preparation and cleanup,
2586 * rather than having multiple threads possibly attempt to run this code
2587 * simultaneously.
2588 *
2589 * @pre assumes the iflib context lock is held and will unlock it while
2590 * waiting for the PF reset to complete.
2591 */
2592 static void
ice_handle_pf_reset_request(struct ice_softc * sc)2593 ice_handle_pf_reset_request(struct ice_softc *sc)
2594 {
2595 struct ice_hw *hw = &sc->hw;
2596 enum ice_status status;
2597
2598 /* Check for PF reset requests */
2599 if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ))
2600 return;
2601
2602 /* Make sure we're prepared for reset */
2603 ice_prepare_for_reset(sc);
2604
2605 /*
2606 * Release the iflib context lock and wait for the device to finish
2607 * resetting.
2608 */
2609 IFLIB_CTX_UNLOCK(sc);
2610 status = ice_reset(hw, ICE_RESET_PFR);
2611 IFLIB_CTX_LOCK(sc);
2612 if (status) {
2613 device_printf(sc->dev, "device PF reset failed, err %s\n",
2614 ice_status_str(status));
2615 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2616 return;
2617 }
2618
2619 sc->soft_stats.pfr_count++;
2620 ice_rebuild(sc);
2621 }
2622
2623 /**
2624 * ice_init_device_features - Init device driver features
2625 * @sc: driver softc structure
2626 *
2627 * @pre assumes that the function capabilities bits have been set up by
2628 * ice_init_hw().
2629 */
2630 static void
ice_init_device_features(struct ice_softc * sc)2631 ice_init_device_features(struct ice_softc *sc)
2632 {
2633 /*
2634 * A failed pkg file download triggers safe mode, disabling advanced
2635 * device feature support
2636 */
2637 if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE))
2638 return;
2639
2640 /* Set capabilities that all devices support */
2641 ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2642 ice_set_bit(ICE_FEATURE_RSS, sc->feat_cap);
2643 ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_cap);
2644 ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_1, sc->feat_cap);
2645 ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_2, sc->feat_cap);
2646 ice_set_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap);
2647 ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap);
2648 ice_set_bit(ICE_FEATURE_HAS_PBA, sc->feat_cap);
2649
2650 /* Disable features due to hardware limitations... */
2651 if (!sc->hw.func_caps.common_cap.rss_table_size)
2652 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap);
2653 /* Disable features due to firmware limitations... */
2654 if (!ice_is_fw_health_report_supported(&sc->hw))
2655 ice_clear_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap);
2656 if (!ice_fwlog_supported(&sc->hw))
2657 ice_clear_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap);
2658 if (sc->hw.fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) {
2659 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_FW_LOGGING))
2660 ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_en);
2661 else
2662 ice_fwlog_unregister(&sc->hw);
2663 }
2664
2665 /* Disable capabilities not supported by the OS */
2666 ice_disable_unsupported_features(sc->feat_cap);
2667
2668 /* RSS is always enabled for iflib */
2669 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RSS))
2670 ice_set_bit(ICE_FEATURE_RSS, sc->feat_en);
2671 }
2672
2673 /**
2674 * ice_if_multi_set - Callback to update Multicast filters in HW
2675 * @ctx: iflib ctx structure
2676 *
2677 * Called by iflib in response to SIOCDELMULTI and SIOCADDMULTI. Must search
2678 * the if_multiaddrs list and determine which filters have been added or
2679 * removed from the list, and update HW programming to reflect the new list.
2680 *
2681 * @pre assumes the caller holds the iflib CTX lock
2682 */
2683 static void
ice_if_multi_set(if_ctx_t ctx)2684 ice_if_multi_set(if_ctx_t ctx)
2685 {
2686 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2687 int err;
2688
2689 ASSERT_CTX_LOCKED(sc);
2690
2691 /* Do not handle multicast configuration in recovery mode */
2692 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2693 return;
2694
2695 err = ice_sync_multicast_filters(sc);
2696 if (err) {
2697 device_printf(sc->dev,
2698 "Failed to synchronize multicast filter list: %s\n",
2699 ice_err_str(err));
2700 return;
2701 }
2702 }
2703
2704 /**
2705 * ice_if_vlan_register - Register a VLAN with the hardware
2706 * @ctx: iflib ctx pointer
2707 * @vtag: VLAN to add
2708 *
2709 * Programs the main PF VSI with a hardware filter for the given VLAN.
2710 *
2711 * @pre assumes the caller holds the iflib CTX lock
2712 */
2713 static void
ice_if_vlan_register(if_ctx_t ctx,u16 vtag)2714 ice_if_vlan_register(if_ctx_t ctx, u16 vtag)
2715 {
2716 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2717 enum ice_status status;
2718
2719 ASSERT_CTX_LOCKED(sc);
2720
2721 /* Do not handle VLAN configuration in recovery mode */
2722 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2723 return;
2724
2725 status = ice_add_vlan_hw_filter(&sc->pf_vsi, vtag);
2726 if (status) {
2727 device_printf(sc->dev,
2728 "Failure adding VLAN %d to main VSI, err %s aq_err %s\n",
2729 vtag, ice_status_str(status),
2730 ice_aq_str(sc->hw.adminq.sq_last_status));
2731 }
2732 }
2733
2734 /**
2735 * ice_if_vlan_unregister - Remove a VLAN filter from the hardware
2736 * @ctx: iflib ctx pointer
2737 * @vtag: VLAN to add
2738 *
2739 * Removes the previously programmed VLAN filter from the main PF VSI.
2740 *
2741 * @pre assumes the caller holds the iflib CTX lock
2742 */
2743 static void
ice_if_vlan_unregister(if_ctx_t ctx,u16 vtag)2744 ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
2745 {
2746 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2747 enum ice_status status;
2748
2749 ASSERT_CTX_LOCKED(sc);
2750
2751 /* Do not handle VLAN configuration in recovery mode */
2752 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2753 return;
2754
2755 status = ice_remove_vlan_hw_filter(&sc->pf_vsi, vtag);
2756 if (status) {
2757 device_printf(sc->dev,
2758 "Failure removing VLAN %d from main VSI, err %s aq_err %s\n",
2759 vtag, ice_status_str(status),
2760 ice_aq_str(sc->hw.adminq.sq_last_status));
2761 }
2762 }
2763
2764 /**
2765 * ice_if_stop - Stop the device
2766 * @ctx: iflib context structure
2767 *
2768 * Called by iflib to stop the device and bring it down. (i.e. ifconfig ice0
2769 * down)
2770 *
2771 * @pre assumes the caller holds the iflib CTX lock
2772 */
2773 static void
ice_if_stop(if_ctx_t ctx)2774 ice_if_stop(if_ctx_t ctx)
2775 {
2776 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2777
2778 ASSERT_CTX_LOCKED(sc);
2779
2780 /*
2781 * The iflib core may call IFDI_STOP prior to the first call to
2782 * IFDI_INIT. This will cause us to attempt to remove MAC filters we
2783 * don't have, and disable Tx queues which aren't yet configured.
2784 * Although it is likely these extra operations are harmless, they do
2785 * cause spurious warning messages to be displayed, which may confuse
2786 * users.
2787 *
2788 * To avoid these messages, we use a state bit indicating if we've
2789 * been initialized. It will be set when ice_if_init is called, and
2790 * cleared here in ice_if_stop.
2791 */
2792 if (!ice_testandclear_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED))
2793 return;
2794
2795 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) {
2796 device_printf(sc->dev, "request to stop interface cannot be completed as the device failed to reset\n");
2797 return;
2798 }
2799
2800 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
2801 device_printf(sc->dev, "request to stop interface while device is prepared for impending reset\n");
2802 return;
2803 }
2804
2805 /* Remove the MAC filters, stop Tx, and stop Rx. We don't check the
2806 * return of these functions because there's nothing we can really do
2807 * if they fail, and the functions already print error messages.
2808 * Just try to shut down as much as we can.
2809 */
2810 ice_rm_pf_default_mac_filters(sc);
2811
2812 /* Dissociate the Tx and Rx queues from the interrupts */
2813 ice_flush_txq_interrupts(&sc->pf_vsi);
2814 ice_flush_rxq_interrupts(&sc->pf_vsi);
2815
2816 /* Disable the Tx and Rx queues */
2817 ice_vsi_disable_tx(&sc->pf_vsi);
2818 ice_control_rx_queues(&sc->pf_vsi, false);
2819 }
2820
2821 /**
2822 * ice_if_get_counter - Get current value of an ifnet statistic
2823 * @ctx: iflib context pointer
2824 * @counter: ifnet counter to read
2825 *
2826 * Reads the current value of an ifnet counter for the device.
2827 *
2828 * This function is not protected by the iflib CTX lock.
2829 */
2830 static uint64_t
ice_if_get_counter(if_ctx_t ctx,ift_counter counter)2831 ice_if_get_counter(if_ctx_t ctx, ift_counter counter)
2832 {
2833 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2834
2835 /* Return the counter for the main PF VSI */
2836 return ice_get_ifnet_counter(&sc->pf_vsi, counter);
2837 }
2838
2839 /**
2840 * ice_request_stack_reinit - Request that iflib re-initialize
2841 * @sc: the device private softc
2842 *
2843 * Request that the device be brought down and up, to re-initialize. For
2844 * example, this may be called when a device reset occurs, or when Tx and Rx
2845 * queues need to be re-initialized.
2846 *
2847 * This is required because the iflib state is outside the driver, and must be
2848 * re-initialized if we need to resart Tx and Rx queues.
2849 */
2850 void
ice_request_stack_reinit(struct ice_softc * sc)2851 ice_request_stack_reinit(struct ice_softc *sc)
2852 {
2853 if (CTX_ACTIVE(sc->ctx)) {
2854 iflib_request_reset(sc->ctx);
2855 iflib_admin_intr_deferred(sc->ctx);
2856 }
2857 }
2858
2859 /**
2860 * ice_driver_is_detaching - Check if the driver is detaching/unloading
2861 * @sc: device private softc
2862 *
2863 * Returns true if the driver is detaching, false otherwise.
2864 *
2865 * @remark on newer kernels, take advantage of iflib_in_detach in order to
2866 * report detachment correctly as early as possible.
2867 *
2868 * @remark this function is used by various code paths that want to avoid
2869 * running if the driver is about to be removed. This includes sysctls and
2870 * other driver access points. Note that it does not fully resolve
2871 * detach-based race conditions as it is possible for a thread to race with
2872 * iflib_in_detach.
2873 */
2874 bool
ice_driver_is_detaching(struct ice_softc * sc)2875 ice_driver_is_detaching(struct ice_softc *sc)
2876 {
2877 return (ice_test_state(&sc->state, ICE_STATE_DETACHING) ||
2878 iflib_in_detach(sc->ctx));
2879 }
2880
2881 /**
2882 * ice_if_priv_ioctl - Device private ioctl handler
2883 * @ctx: iflib context pointer
2884 * @command: The ioctl command issued
2885 * @data: ioctl specific data
2886 *
2887 * iflib callback for handling custom driver specific ioctls.
2888 *
2889 * @pre Assumes that the iflib context lock is held.
2890 */
2891 static int
ice_if_priv_ioctl(if_ctx_t ctx,u_long command,caddr_t data)2892 ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data)
2893 {
2894 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2895 struct ifdrv *ifd;
2896 device_t dev = sc->dev;
2897
2898 if (data == NULL)
2899 return (EINVAL);
2900
2901 ASSERT_CTX_LOCKED(sc);
2902
2903 /* Make sure the command type is valid */
2904 switch (command) {
2905 case SIOCSDRVSPEC:
2906 case SIOCGDRVSPEC:
2907 /* Accepted commands */
2908 break;
2909 case SIOCGPRIVATE_0:
2910 /*
2911 * Although we do not support this ioctl command, it's
2912 * expected that iflib will forward it to the IFDI_PRIV_IOCTL
2913 * handler. Do not print a message in this case
2914 */
2915 return (ENOTSUP);
2916 default:
2917 /*
2918 * If we get a different command for this function, it's
2919 * definitely unexpected, so log a message indicating what
2920 * command we got for debugging purposes.
2921 */
2922 device_printf(dev, "%s: unexpected ioctl command %08lx\n",
2923 __func__, command);
2924 return (EINVAL);
2925 }
2926
2927 ifd = (struct ifdrv *)data;
2928
2929 switch (ifd->ifd_cmd) {
2930 case ICE_NVM_ACCESS:
2931 return ice_handle_nvm_access_ioctl(sc, ifd);
2932 default:
2933 return EINVAL;
2934 }
2935 }
2936
2937 /**
2938 * ice_if_i2c_req - I2C request handler for iflib
2939 * @ctx: iflib context pointer
2940 * @req: The I2C parameters to use
2941 *
2942 * Read from the port's I2C eeprom using the parameters from the ioctl.
2943 *
2944 * @remark The iflib-only part is pretty simple.
2945 */
2946 static int
ice_if_i2c_req(if_ctx_t ctx,struct ifi2creq * req)2947 ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req)
2948 {
2949 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2950
2951 return ice_handle_i2c_req(sc, req);
2952 }
2953
2954 /**
2955 * ice_if_suspend - PCI device suspend handler for iflib
2956 * @ctx: iflib context pointer
2957 *
2958 * Deinitializes the driver and clears HW resources in preparation for
2959 * suspend or an FLR.
2960 *
2961 * @returns 0; this return value is ignored
2962 */
2963 static int
ice_if_suspend(if_ctx_t ctx)2964 ice_if_suspend(if_ctx_t ctx)
2965 {
2966 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2967
2968 /* At least a PFR is always going to happen after this;
2969 * either via FLR or during the D3->D0 transition.
2970 */
2971 ice_clear_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
2972
2973 ice_prepare_for_reset(sc);
2974
2975 return (0);
2976 }
2977
2978 /**
2979 * ice_if_resume - PCI device resume handler for iflib
2980 * @ctx: iflib context pointer
2981 *
2982 * Reinitializes the driver and the HW after PCI resume or after
2983 * an FLR. An init is performed by iflib after this function is finished.
2984 *
2985 * @returns 0; this return value is ignored
2986 */
2987 static int
ice_if_resume(if_ctx_t ctx)2988 ice_if_resume(if_ctx_t ctx)
2989 {
2990 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2991
2992 ice_rebuild(sc);
2993
2994 return (0);
2995 }
2996
2997