Lines Matching refs:fl
841 iflib_fl_t fl = &rxq->ifr_fl[0]; in iflib_netmap_config() local
847 info->rx_buf_maxsize = fl->ifl_buf_size; in iflib_netmap_config()
864 iflib_fl_t fl = &rxq->ifr_fl[0]; in netmap_fl_refill() local
899 map = fl->ifl_sds.ifsd_map; in netmap_fl_refill()
900 nic_i = fl->ifl_pidx; in netmap_fl_refill()
920 void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[i]); in netmap_fl_refill()
927 fl->ifl_rxd_idxs[i] = nic_i; in netmap_fl_refill()
930 netmap_load_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
934 netmap_reload_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
937 bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i], in netmap_fl_refill()
949 fl->ifl_pidx = nic_i; in netmap_fl_refill()
957 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in netmap_fl_refill()
959 ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, in netmap_fl_refill()
1177 iflib_fl_t fl = &rxq->ifr_fl[0]; in iflib_netmap_rxsync() local
1186 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_netmap_rxsync()
1218 cidxp = &fl->ifl_cidx; in iflib_netmap_rxsync()
1222 nic_i = fl->ifl_cidx; in iflib_netmap_rxsync()
1250 bus_dmamap_sync(fl->ifl_buf_tag, in iflib_netmap_rxsync()
1251 fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); in iflib_netmap_rxsync()
1253 fl->ifl_cidx = nic_i = nm_next(nic_i, lim); in iflib_netmap_rxsync()
1414 iflib_fl_t fl; in iru_init() local
1416 fl = &rxq->ifr_fl[flid]; in iru_init()
1417 iru->iru_paddrs = fl->ifl_bus_addrs; in iru_init()
1418 iru->iru_idxs = fl->ifl_rxd_idxs; in iru_init()
1420 iru->iru_buf_size = fl->ifl_buf_size; in iru_init()
1421 iru->iru_flidx = fl->ifl_id; in iru_init()
1931 iflib_fl_t fl; in iflib_rxsd_alloc() local
1937 fl = rxq->ifr_fl; in iflib_rxsd_alloc()
1938 for (int i = 0; i < rxq->ifr_nfl; i++, fl++) { in iflib_rxsd_alloc()
1939 fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */ in iflib_rxsd_alloc()
1952 &fl->ifl_buf_tag); in iflib_rxsd_alloc()
1960 if (!(fl->ifl_sds.ifsd_m = in iflib_rxsd_alloc()
1970 if (!(fl->ifl_sds.ifsd_cl = in iflib_rxsd_alloc()
1980 if (!(fl->ifl_sds.ifsd_ba = in iflib_rxsd_alloc()
1992 if (!(fl->ifl_sds.ifsd_map = in iflib_rxsd_alloc()
2000 err = bus_dmamap_create(fl->ifl_buf_tag, 0, in iflib_rxsd_alloc()
2001 &fl->ifl_sds.ifsd_map[i]); in iflib_rxsd_alloc()
2046 iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) in iflib_fl_refill() argument
2058 MPASS(count <= fl->ifl_size - fl->ifl_credits - 1); in iflib_fl_refill()
2060 sd_m = fl->ifl_sds.ifsd_m; in iflib_fl_refill()
2061 sd_map = fl->ifl_sds.ifsd_map; in iflib_fl_refill()
2062 sd_cl = fl->ifl_sds.ifsd_cl; in iflib_fl_refill()
2063 sd_ba = fl->ifl_sds.ifsd_ba; in iflib_fl_refill()
2064 pidx = fl->ifl_pidx; in iflib_fl_refill()
2066 frag_idx = fl->ifl_fragidx; in iflib_fl_refill()
2067 credits = fl->ifl_credits; in iflib_fl_refill()
2072 MPASS(credits + n <= fl->ifl_size); in iflib_fl_refill()
2074 if (pidx < fl->ifl_cidx) in iflib_fl_refill()
2075 MPASS(pidx + n <= fl->ifl_cidx); in iflib_fl_refill()
2076 if (pidx == fl->ifl_cidx && (credits < fl->ifl_size)) in iflib_fl_refill()
2077 MPASS(fl->ifl_gen == 0); in iflib_fl_refill()
2078 if (pidx > fl->ifl_cidx) in iflib_fl_refill()
2079 MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx); in iflib_fl_refill()
2084 iru_init(&iru, fl->ifl_rxq, fl->ifl_id); in iflib_fl_refill()
2093 bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size, in iflib_fl_refill()
2096 bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); in iflib_fl_refill()
2099 cl = uma_zalloc(fl->ifl_zone, M_NOWAIT); in iflib_fl_refill()
2105 err = bus_dmamap_load(fl->ifl_buf_tag, sd_map[frag_idx], in iflib_fl_refill()
2106 cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, in iflib_fl_refill()
2109 uma_zfree(fl->ifl_zone, cl); in iflib_fl_refill()
2116 fl->ifl_cl_enqueued++; in iflib_fl_refill()
2121 bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx], in iflib_fl_refill()
2130 bit_set(fl->ifl_rx_bitmap, frag_idx); in iflib_fl_refill()
2132 fl->ifl_m_enqueued++; in iflib_fl_refill()
2136 fl->ifl_rxd_idxs[i] = frag_idx; in iflib_fl_refill()
2137 fl->ifl_bus_addrs[i] = bus_addr; in iflib_fl_refill()
2140 MPASS(credits <= fl->ifl_size); in iflib_fl_refill()
2141 if (++idx == fl->ifl_size) { in iflib_fl_refill()
2143 fl->ifl_gen = 1; in iflib_fl_refill()
2151 fl->ifl_pidx = idx; in iflib_fl_refill()
2152 fl->ifl_credits = credits; in iflib_fl_refill()
2163 fl->ifl_pidx = idx; in iflib_fl_refill()
2164 fl->ifl_credits = credits; in iflib_fl_refill()
2167 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_fl_refill()
2169 ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, in iflib_fl_refill()
2170 fl->ifl_id, fl->ifl_pidx); in iflib_fl_refill()
2171 if (__predict_true(bit_test(fl->ifl_rx_bitmap, frag_idx))) { in iflib_fl_refill()
2172 fl->ifl_fragidx = frag_idx + 1; in iflib_fl_refill()
2173 if (fl->ifl_fragidx == fl->ifl_size) in iflib_fl_refill()
2174 fl->ifl_fragidx = 0; in iflib_fl_refill()
2176 fl->ifl_fragidx = frag_idx; in iflib_fl_refill()
2184 iflib_fl_refill_all(if_ctx_t ctx, iflib_fl_t fl) in iflib_fl_refill_all() argument
2197 int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1; in iflib_fl_refill_all()
2199 …int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - … in iflib_fl_refill_all()
2202 MPASS(fl->ifl_credits <= fl->ifl_size); in iflib_fl_refill_all()
2206 return (iflib_fl_refill(ctx, fl, reclaimable)); in iflib_fl_refill_all()
2222 iflib_fl_bufs_free(iflib_fl_t fl) in iflib_fl_bufs_free() argument
2224 iflib_dma_info_t idi = fl->ifl_ifdi; in iflib_fl_bufs_free()
2228 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2229 struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; in iflib_fl_bufs_free()
2230 caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; in iflib_fl_bufs_free()
2233 sd_map = fl->ifl_sds.ifsd_map[i]; in iflib_fl_bufs_free()
2234 bus_dmamap_sync(fl->ifl_buf_tag, sd_map, in iflib_fl_bufs_free()
2236 bus_dmamap_unload(fl->ifl_buf_tag, sd_map); in iflib_fl_bufs_free()
2237 uma_zfree(fl->ifl_zone, *sd_cl); in iflib_fl_bufs_free()
2248 fl->ifl_m_dequeued++; in iflib_fl_bufs_free()
2249 fl->ifl_cl_dequeued++; in iflib_fl_bufs_free()
2253 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2254 MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); in iflib_fl_bufs_free()
2255 MPASS(fl->ifl_sds.ifsd_m[i] == NULL); in iflib_fl_bufs_free()
2261 fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0; in iflib_fl_bufs_free()
2271 iflib_fl_setup(iflib_fl_t fl) in iflib_fl_setup() argument
2273 iflib_rxq_t rxq = fl->ifl_rxq; in iflib_fl_setup()
2278 bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1); in iflib_fl_setup()
2282 iflib_fl_bufs_free(fl); in iflib_fl_setup()
2284 MPASS(fl->ifl_credits == 0); in iflib_fl_setup()
2285 qidx = rxq->ifr_fl_offset + fl->ifl_id; in iflib_fl_setup()
2287 fl->ifl_buf_size = scctx->isc_rxd_buf_size[qidx]; in iflib_fl_setup()
2289 fl->ifl_buf_size = ctx->ifc_rx_mbuf_sz; in iflib_fl_setup()
2294 fl->ifl_buf_size = iflib_get_mbuf_size_for(fl->ifl_buf_size); in iflib_fl_setup()
2295 if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size) in iflib_fl_setup()
2296 ctx->ifc_max_fl_buf_size = fl->ifl_buf_size; in iflib_fl_setup()
2297 fl->ifl_cltype = m_gettype(fl->ifl_buf_size); in iflib_fl_setup()
2298 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_fl_setup()
2306 MPASS(fl->ifl_size > 0); in iflib_fl_setup()
2307 (void)iflib_fl_refill(ctx, fl, min(128, fl->ifl_size - 1)); in iflib_fl_setup()
2308 if (min(128, fl->ifl_size - 1) != fl->ifl_credits) in iflib_fl_setup()
2314 MPASS(fl->ifl_ifdi != NULL); in iflib_fl_setup()
2315 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_fl_setup()
2328 iflib_fl_t fl; in iflib_rx_sds_free() local
2333 fl = &rxq->ifr_fl[i]; in iflib_rx_sds_free()
2334 if (fl->ifl_buf_tag != NULL) { in iflib_rx_sds_free()
2335 if (fl->ifl_sds.ifsd_map != NULL) { in iflib_rx_sds_free()
2336 for (j = 0; j < fl->ifl_size; j++) { in iflib_rx_sds_free()
2338 fl->ifl_buf_tag, in iflib_rx_sds_free()
2339 fl->ifl_sds.ifsd_map[j], in iflib_rx_sds_free()
2342 fl->ifl_buf_tag, in iflib_rx_sds_free()
2343 fl->ifl_sds.ifsd_map[j]); in iflib_rx_sds_free()
2345 fl->ifl_buf_tag, in iflib_rx_sds_free()
2346 fl->ifl_sds.ifsd_map[j]); in iflib_rx_sds_free()
2349 bus_dma_tag_destroy(fl->ifl_buf_tag); in iflib_rx_sds_free()
2350 fl->ifl_buf_tag = NULL; in iflib_rx_sds_free()
2352 free(fl->ifl_sds.ifsd_m, M_IFLIB); in iflib_rx_sds_free()
2353 free(fl->ifl_sds.ifsd_cl, M_IFLIB); in iflib_rx_sds_free()
2354 free(fl->ifl_sds.ifsd_ba, M_IFLIB); in iflib_rx_sds_free()
2355 free(fl->ifl_sds.ifsd_map, M_IFLIB); in iflib_rx_sds_free()
2356 free(fl->ifl_rx_bitmap, M_IFLIB); in iflib_rx_sds_free()
2357 fl->ifl_sds.ifsd_m = NULL; in iflib_rx_sds_free()
2358 fl->ifl_sds.ifsd_cl = NULL; in iflib_rx_sds_free()
2359 fl->ifl_sds.ifsd_ba = NULL; in iflib_rx_sds_free()
2360 fl->ifl_sds.ifsd_map = NULL; in iflib_rx_sds_free()
2361 fl->ifl_rx_bitmap = NULL; in iflib_rx_sds_free()
2463 iflib_fl_t fl; in iflib_init_locked() local
2517 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_init_locked()
2518 if (iflib_fl_setup(fl)) { in iflib_init_locked()
2570 iflib_fl_t fl; in iflib_stop() local
2623 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_stop()
2624 iflib_fl_bufs_free(fl); in iflib_stop()
2629 calc_next_rxd(iflib_fl_t fl, int cidx) in calc_next_rxd() argument
2635 nrxd = fl->ifl_size; in calc_next_rxd()
2636 size = fl->ifl_rxd_size; in calc_next_rxd()
2637 start = fl->ifl_ifdi->idi_vaddr; in calc_next_rxd()
2648 prefetch_pkts(iflib_fl_t fl, int cidx) in prefetch_pkts() argument
2651 int nrxd = fl->ifl_size; in prefetch_pkts()
2655 prefetch(&fl->ifl_sds.ifsd_m[nextptr]); in prefetch_pkts()
2656 prefetch(&fl->ifl_sds.ifsd_cl[nextptr]); in prefetch_pkts()
2657 next_rxd = calc_next_rxd(fl, cidx); in prefetch_pkts()
2659 prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd-1)]); in prefetch_pkts()
2660 prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd-1)]); in prefetch_pkts()
2661 prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd-1)]); in prefetch_pkts()
2662 prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd-1)]); in prefetch_pkts()
2663 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd-1)]); in prefetch_pkts()
2664 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd-1)]); in prefetch_pkts()
2665 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd-1)]); in prefetch_pkts()
2666 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd-1)]); in prefetch_pkts()
2674 iflib_fl_t fl; in rxd_frag_to_sd() local
2682 fl = &rxq->ifr_fl[flid]; in rxd_frag_to_sd()
2683 sd->ifsd_fl = fl; in rxd_frag_to_sd()
2684 m = fl->ifl_sds.ifsd_m[cidx]; in rxd_frag_to_sd()
2685 sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; in rxd_frag_to_sd()
2686 fl->ifl_credits--; in rxd_frag_to_sd()
2688 fl->ifl_m_dequeued++; in rxd_frag_to_sd()
2691 prefetch_pkts(fl, cidx); in rxd_frag_to_sd()
2692 next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); in rxd_frag_to_sd()
2693 prefetch(&fl->ifl_sds.ifsd_map[next]); in rxd_frag_to_sd()
2694 map = fl->ifl_sds.ifsd_map[cidx]; in rxd_frag_to_sd()
2696 bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); in rxd_frag_to_sd()
2726 fl->ifl_sds.ifsd_m[cidx] = NULL; in rxd_frag_to_sd()
2732 fl->ifl_sds.ifsd_m[cidx] = NULL; in rxd_frag_to_sd()
2738 bus_dmamap_unload(fl->ifl_buf_tag, map); in rxd_frag_to_sd()
2739 fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); in rxd_frag_to_sd()
2740 if (__predict_false(fl->ifl_cidx == 0)) in rxd_frag_to_sd()
2741 fl->ifl_gen = 0; in rxd_frag_to_sd()
2742 bit_clear(fl->ifl_rx_bitmap, cidx); in rxd_frag_to_sd()
2919 iflib_fl_t fl; in iflib_rxeof() local
2942 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
2943 retval |= iflib_fl_refill_all(ctx, fl); in iflib_rxeof()
3002 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
3003 retval |= iflib_fl_refill_all(ctx, fl); in iflib_rxeof()
5897 iflib_fl_t fl = NULL; in iflib_queues_alloc() local
6022 if (!(fl = in iflib_queues_alloc()
6028 rxq->ifr_fl = fl; in iflib_queues_alloc()
6030 fl[j].ifl_rxq = rxq; in iflib_queues_alloc()
6031 fl[j].ifl_id = j; in iflib_queues_alloc()
6032 fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset]; in iflib_queues_alloc()
6033 fl[j].ifl_rxd_size = scctx->isc_rxd_size[j]; in iflib_queues_alloc()
6043 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_queues_alloc()
6044 fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB, in iflib_queues_alloc()
6556 iflib_fl_t fl; in iflib_rxd_avail() local
6559 for (i = 0, fl = &rxq->ifr_fl[0]; i < rxq->ifr_nfl; i++, fl++) in iflib_rxd_avail()
6560 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_rxd_avail()
6893 iflib_fl_t fl; in iflib_add_device_sysctl_post() local
7011 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_add_device_sysctl_post()
7018 &fl->ifl_pidx, 1, "Producer Index"); in iflib_add_device_sysctl_post()
7021 &fl->ifl_cidx, 1, "Consumer Index"); in iflib_add_device_sysctl_post()
7024 &fl->ifl_credits, 1, "credits available"); in iflib_add_device_sysctl_post()
7027 &fl->ifl_buf_size, 1, "buffer size"); in iflib_add_device_sysctl_post()
7031 &fl->ifl_m_enqueued, "mbufs allocated"); in iflib_add_device_sysctl_post()
7034 &fl->ifl_m_dequeued, "mbufs freed"); in iflib_add_device_sysctl_post()
7037 &fl->ifl_cl_enqueued, "clusters allocated"); in iflib_add_device_sysctl_post()
7040 &fl->ifl_cl_dequeued, "clusters freed"); in iflib_add_device_sysctl_post()
7102 iflib_fl_t fl; in iflib_debugnet_event() local
7114 fl = rxq->ifr_fl; in iflib_debugnet_event()
7115 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_debugnet_event()