Lines Matching refs:fl
817 iflib_fl_t fl = &rxq->ifr_fl[0]; in netmap_fl_refill() local
826 map = fl->ifl_sds.ifsd_map; in netmap_fl_refill()
842 void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[tmp_pidx]); in netmap_fl_refill()
851 fl->ifl_vm_addrs[tmp_pidx] = addr; in netmap_fl_refill()
853 netmap_load_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
857 netmap_reload_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
863 fl->ifl_rxd_idxs[tmp_pidx] = nic_i = nm_next(nic_i, lim); in netmap_fl_refill()
872 bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i_dma], in netmap_fl_refill()
881 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in netmap_fl_refill()
884 ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i); in netmap_fl_refill()
1057 iflib_fl_t fl; in iflib_netmap_rxsync() local
1075 for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) { in iflib_netmap_rxsync()
1076 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_netmap_rxsync()
1100 fl = &rxq->ifr_fl[i]; in iflib_netmap_rxsync()
1101 nic_i = fl->ifl_cidx; in iflib_netmap_rxsync()
1115 bus_dmamap_sync(fl->ifl_buf_tag, in iflib_netmap_rxsync()
1116 fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); in iflib_netmap_rxsync()
1126 fl->ifl_cidx = nic_i; in iflib_netmap_rxsync()
1315 iflib_fl_t fl; in iru_init() local
1317 fl = &rxq->ifr_fl[flid]; in iru_init()
1318 iru->iru_paddrs = fl->ifl_bus_addrs; in iru_init()
1319 iru->iru_vaddrs = &fl->ifl_vm_addrs[0]; in iru_init()
1320 iru->iru_idxs = fl->ifl_rxd_idxs; in iru_init()
1322 iru->iru_buf_size = fl->ifl_buf_size; in iru_init()
1323 iru->iru_flidx = fl->ifl_id; in iru_init()
1860 iflib_fl_t fl; in iflib_rxsd_alloc() local
1866 fl = rxq->ifr_fl; in iflib_rxsd_alloc()
1867 for (int i = 0; i < rxq->ifr_nfl; i++, fl++) { in iflib_rxsd_alloc()
1868 fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */ in iflib_rxsd_alloc()
1881 &fl->ifl_buf_tag); in iflib_rxsd_alloc()
1889 if (!(fl->ifl_sds.ifsd_m = in iflib_rxsd_alloc()
1899 if (!(fl->ifl_sds.ifsd_cl = in iflib_rxsd_alloc()
1909 if (!(fl->ifl_sds.ifsd_ba = in iflib_rxsd_alloc()
1921 if (!(fl->ifl_sds.ifsd_map = in iflib_rxsd_alloc()
1929 err = bus_dmamap_create(fl->ifl_buf_tag, 0, in iflib_rxsd_alloc()
1930 &fl->ifl_sds.ifsd_map[i]); in iflib_rxsd_alloc()
1975 _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) in _iflib_fl_refill() argument
1987 sd_m = fl->ifl_sds.ifsd_m; in _iflib_fl_refill()
1988 sd_map = fl->ifl_sds.ifsd_map; in _iflib_fl_refill()
1989 sd_cl = fl->ifl_sds.ifsd_cl; in _iflib_fl_refill()
1990 sd_ba = fl->ifl_sds.ifsd_ba; in _iflib_fl_refill()
1991 pidx = fl->ifl_pidx; in _iflib_fl_refill()
1993 frag_idx = fl->ifl_fragidx; in _iflib_fl_refill()
1994 credits = fl->ifl_credits; in _iflib_fl_refill()
1999 MPASS(credits + n <= fl->ifl_size); in _iflib_fl_refill()
2001 if (pidx < fl->ifl_cidx) in _iflib_fl_refill()
2002 MPASS(pidx + n <= fl->ifl_cidx); in _iflib_fl_refill()
2003 if (pidx == fl->ifl_cidx && (credits < fl->ifl_size)) in _iflib_fl_refill()
2004 MPASS(fl->ifl_gen == 0); in _iflib_fl_refill()
2005 if (pidx > fl->ifl_cidx) in _iflib_fl_refill()
2006 MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx); in _iflib_fl_refill()
2011 iru_init(&iru, fl->ifl_rxq, fl->ifl_id); in _iflib_fl_refill()
2019 bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size, in _iflib_fl_refill()
2022 bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); in _iflib_fl_refill()
2025 if ((cl = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL) in _iflib_fl_refill()
2030 err = bus_dmamap_load(fl->ifl_buf_tag, sd_map[frag_idx], in _iflib_fl_refill()
2031 cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, in _iflib_fl_refill()
2037 if (fl->ifl_zone == zone_pack) in _iflib_fl_refill()
2038 uma_zfree(fl->ifl_zone, cl); in _iflib_fl_refill()
2045 fl->ifl_cl_enqueued++; in _iflib_fl_refill()
2050 bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx], in _iflib_fl_refill()
2058 bit_set(fl->ifl_rx_bitmap, frag_idx); in _iflib_fl_refill()
2060 fl->ifl_m_enqueued++; in _iflib_fl_refill()
2064 fl->ifl_rxd_idxs[i] = frag_idx; in _iflib_fl_refill()
2065 fl->ifl_bus_addrs[i] = bus_addr; in _iflib_fl_refill()
2066 fl->ifl_vm_addrs[i] = cl; in _iflib_fl_refill()
2069 MPASS(credits <= fl->ifl_size); in _iflib_fl_refill()
2070 if (++idx == fl->ifl_size) { in _iflib_fl_refill()
2071 fl->ifl_gen = 1; in _iflib_fl_refill()
2080 fl->ifl_pidx = idx; in _iflib_fl_refill()
2081 fl->ifl_credits = credits; in _iflib_fl_refill()
2089 fl->ifl_pidx = idx; in _iflib_fl_refill()
2090 fl->ifl_credits = credits; in _iflib_fl_refill()
2093 if (fl->ifl_pidx == 0) in _iflib_fl_refill()
2094 pidx = fl->ifl_size - 1; in _iflib_fl_refill()
2096 pidx = fl->ifl_pidx - 1; in _iflib_fl_refill()
2098 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in _iflib_fl_refill()
2100 ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx); in _iflib_fl_refill()
2101 fl->ifl_fragidx = frag_idx; in _iflib_fl_refill()
2105 __iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max) in __iflib_fl_refill_lt() argument
2108 int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1; in __iflib_fl_refill_lt()
2110 …int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - … in __iflib_fl_refill_lt()
2113 MPASS(fl->ifl_credits <= fl->ifl_size); in __iflib_fl_refill_lt()
2117 _iflib_fl_refill(ctx, fl, min(max, reclaimable)); in __iflib_fl_refill_lt()
2132 iflib_fl_bufs_free(iflib_fl_t fl) in iflib_fl_bufs_free() argument
2134 iflib_dma_info_t idi = fl->ifl_ifdi; in iflib_fl_bufs_free()
2138 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2139 struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; in iflib_fl_bufs_free()
2140 caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; in iflib_fl_bufs_free()
2143 sd_map = fl->ifl_sds.ifsd_map[i]; in iflib_fl_bufs_free()
2144 bus_dmamap_sync(fl->ifl_buf_tag, sd_map, in iflib_fl_bufs_free()
2146 bus_dmamap_unload(fl->ifl_buf_tag, sd_map); in iflib_fl_bufs_free()
2148 uma_zfree(fl->ifl_zone, *sd_cl); in iflib_fl_bufs_free()
2150 if (iflib_in_detach(fl->ifl_rxq->ifr_ctx)) in iflib_fl_bufs_free()
2151 bus_dmamap_destroy(fl->ifl_buf_tag, sd_map); in iflib_fl_bufs_free()
2161 fl->ifl_m_dequeued++; in iflib_fl_bufs_free()
2162 fl->ifl_cl_dequeued++; in iflib_fl_bufs_free()
2168 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2169 MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); in iflib_fl_bufs_free()
2170 MPASS(fl->ifl_sds.ifsd_m[i] == NULL); in iflib_fl_bufs_free()
2176 fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0; in iflib_fl_bufs_free()
2186 iflib_fl_setup(iflib_fl_t fl) in iflib_fl_setup() argument
2188 iflib_rxq_t rxq = fl->ifl_rxq; in iflib_fl_setup()
2191 bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1); in iflib_fl_setup()
2195 iflib_fl_bufs_free(fl); in iflib_fl_setup()
2197 MPASS(fl->ifl_credits == 0); in iflib_fl_setup()
2198 fl->ifl_buf_size = ctx->ifc_rx_mbuf_sz; in iflib_fl_setup()
2199 if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size) in iflib_fl_setup()
2200 ctx->ifc_max_fl_buf_size = fl->ifl_buf_size; in iflib_fl_setup()
2201 fl->ifl_cltype = m_gettype(fl->ifl_buf_size); in iflib_fl_setup()
2202 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_fl_setup()
2208 _iflib_fl_refill(ctx, fl, min(128, fl->ifl_size)); in iflib_fl_setup()
2209 MPASS(min(128, fl->ifl_size) == fl->ifl_credits); in iflib_fl_setup()
2210 if (min(128, fl->ifl_size) != fl->ifl_credits) in iflib_fl_setup()
2216 MPASS(fl->ifl_ifdi != NULL); in iflib_fl_setup()
2217 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_fl_setup()
2230 iflib_fl_t fl; in iflib_rx_sds_free() local
2235 fl = &rxq->ifr_fl[i]; in iflib_rx_sds_free()
2236 if (fl->ifl_buf_tag != NULL) { in iflib_rx_sds_free()
2237 if (fl->ifl_sds.ifsd_map != NULL) { in iflib_rx_sds_free()
2238 for (j = 0; j < fl->ifl_size; j++) { in iflib_rx_sds_free()
2239 if (fl->ifl_sds.ifsd_map[j] == in iflib_rx_sds_free()
2243 fl->ifl_buf_tag, in iflib_rx_sds_free()
2244 fl->ifl_sds.ifsd_map[j], in iflib_rx_sds_free()
2247 fl->ifl_buf_tag, in iflib_rx_sds_free()
2248 fl->ifl_sds.ifsd_map[j]); in iflib_rx_sds_free()
2251 bus_dma_tag_destroy(fl->ifl_buf_tag); in iflib_rx_sds_free()
2252 fl->ifl_buf_tag = NULL; in iflib_rx_sds_free()
2254 free(fl->ifl_sds.ifsd_m, M_IFLIB); in iflib_rx_sds_free()
2255 free(fl->ifl_sds.ifsd_cl, M_IFLIB); in iflib_rx_sds_free()
2256 free(fl->ifl_sds.ifsd_ba, M_IFLIB); in iflib_rx_sds_free()
2257 free(fl->ifl_sds.ifsd_map, M_IFLIB); in iflib_rx_sds_free()
2258 fl->ifl_sds.ifsd_m = NULL; in iflib_rx_sds_free()
2259 fl->ifl_sds.ifsd_cl = NULL; in iflib_rx_sds_free()
2260 fl->ifl_sds.ifsd_ba = NULL; in iflib_rx_sds_free()
2261 fl->ifl_sds.ifsd_map = NULL; in iflib_rx_sds_free()
2356 iflib_fl_t fl; in iflib_init_locked() local
2403 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_init_locked()
2404 if (iflib_fl_setup(fl)) { in iflib_init_locked()
2453 iflib_fl_t fl; in iflib_stop() local
2495 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_stop()
2496 iflib_fl_bufs_free(fl); in iflib_stop()
2501 calc_next_rxd(iflib_fl_t fl, int cidx) in calc_next_rxd() argument
2507 nrxd = fl->ifl_size; in calc_next_rxd()
2508 size = fl->ifl_rxd_size; in calc_next_rxd()
2509 start = fl->ifl_ifdi->idi_vaddr; in calc_next_rxd()
2520 prefetch_pkts(iflib_fl_t fl, int cidx) in prefetch_pkts() argument
2523 int nrxd = fl->ifl_size; in prefetch_pkts()
2528 prefetch(&fl->ifl_sds.ifsd_m[nextptr]); in prefetch_pkts()
2529 prefetch(&fl->ifl_sds.ifsd_cl[nextptr]); in prefetch_pkts()
2530 next_rxd = calc_next_rxd(fl, cidx); in prefetch_pkts()
2532 prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd-1)]); in prefetch_pkts()
2533 prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd-1)]); in prefetch_pkts()
2534 prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd-1)]); in prefetch_pkts()
2535 prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd-1)]); in prefetch_pkts()
2536 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd-1)]); in prefetch_pkts()
2537 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd-1)]); in prefetch_pkts()
2538 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd-1)]); in prefetch_pkts()
2539 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd-1)]); in prefetch_pkts()
2547 iflib_fl_t fl; in rxd_frag_to_sd() local
2553 fl = &rxq->ifr_fl[flid]; in rxd_frag_to_sd()
2554 sd->ifsd_fl = fl; in rxd_frag_to_sd()
2556 sd->ifsd_m = &fl->ifl_sds.ifsd_m[cidx]; in rxd_frag_to_sd()
2557 sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; in rxd_frag_to_sd()
2558 fl->ifl_credits--; in rxd_frag_to_sd()
2560 fl->ifl_m_dequeued++; in rxd_frag_to_sd()
2563 prefetch_pkts(fl, cidx); in rxd_frag_to_sd()
2564 next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); in rxd_frag_to_sd()
2565 prefetch(&fl->ifl_sds.ifsd_map[next]); in rxd_frag_to_sd()
2566 map = fl->ifl_sds.ifsd_map[cidx]; in rxd_frag_to_sd()
2567 next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1); in rxd_frag_to_sd()
2570 MPASS(fl->ifl_cidx == cidx); in rxd_frag_to_sd()
2571 bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); in rxd_frag_to_sd()
2573 bus_dmamap_unload(fl->ifl_buf_tag, map); in rxd_frag_to_sd()
2574 fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); in rxd_frag_to_sd()
2575 if (__predict_false(fl->ifl_cidx == 0)) in rxd_frag_to_sd()
2576 fl->ifl_gen = 0; in rxd_frag_to_sd()
2577 bit_clear(fl->ifl_rx_bitmap, cidx); in rxd_frag_to_sd()
2727 iflib_fl_t fl; in iflib_rxeof() local
2747 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
2748 __iflib_fl_refill_lt(ctx, fl, budget + 8); in iflib_rxeof()
2803 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
2804 __iflib_fl_refill_lt(ctx, fl, budget + 8); in iflib_rxeof()
4955 iflib_fl_t fl; in iflib_pseudo_deregister() local
4969 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_pseudo_deregister()
4970 free(fl->ifl_rx_bitmap, M_IFLIB); in iflib_pseudo_deregister()
5012 iflib_fl_t fl; in iflib_device_deregister() local
5049 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_device_deregister()
5050 free(fl->ifl_rx_bitmap, M_IFLIB); in iflib_device_deregister()
5361 iflib_fl_t fl = NULL; in iflib_queues_alloc() local
5483 if (!(fl = in iflib_queues_alloc()
5489 rxq->ifr_fl = fl; in iflib_queues_alloc()
5491 fl[j].ifl_rxq = rxq; in iflib_queues_alloc()
5492 fl[j].ifl_id = j; in iflib_queues_alloc()
5493 fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset]; in iflib_queues_alloc()
5494 fl[j].ifl_rxd_size = scctx->isc_rxd_size[j]; in iflib_queues_alloc()
5504 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_queues_alloc()
5505 fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB, in iflib_queues_alloc()
6150 iflib_fl_t fl; in iflib_rxd_avail() local
6153 for (i = 0, fl = &rxq->ifr_fl[0]; i < rxq->ifr_nfl; i++, fl++) in iflib_rxd_avail()
6154 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_rxd_avail()
6484 iflib_fl_t fl; in iflib_add_device_sysctl_post() local
6595 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_add_device_sysctl_post()
6602 &fl->ifl_pidx, 1, "Producer Index"); in iflib_add_device_sysctl_post()
6605 &fl->ifl_cidx, 1, "Consumer Index"); in iflib_add_device_sysctl_post()
6608 &fl->ifl_credits, 1, "credits available"); in iflib_add_device_sysctl_post()
6612 &fl->ifl_m_enqueued, "mbufs allocated"); in iflib_add_device_sysctl_post()
6615 &fl->ifl_m_dequeued, "mbufs freed"); in iflib_add_device_sysctl_post()
6618 &fl->ifl_cl_enqueued, "clusters allocated"); in iflib_add_device_sysctl_post()
6621 &fl->ifl_cl_dequeued, "clusters freed"); in iflib_add_device_sysctl_post()
6684 iflib_fl_t fl; in iflib_netdump_event() local
6696 fl = rxq->ifr_fl; in iflib_netdump_event()
6697 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_netdump_event()