Lines Matching refs:fl
841 iflib_fl_t fl = &rxq->ifr_fl[0]; in netmap_fl_refill() local
876 map = fl->ifl_sds.ifsd_map; in netmap_fl_refill()
877 nic_i = fl->ifl_pidx; in netmap_fl_refill()
897 void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[i]); in netmap_fl_refill()
904 fl->ifl_rxd_idxs[i] = nic_i; in netmap_fl_refill()
907 netmap_load_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
911 netmap_reload_map(na, fl->ifl_buf_tag, in netmap_fl_refill()
914 bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i], in netmap_fl_refill()
926 fl->ifl_pidx = nic_i; in netmap_fl_refill()
934 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in netmap_fl_refill()
936 ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, in netmap_fl_refill()
1145 iflib_fl_t fl = &rxq->ifr_fl[0]; in iflib_netmap_rxsync() local
1154 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_netmap_rxsync()
1186 cidxp = &fl->ifl_cidx; in iflib_netmap_rxsync()
1190 nic_i = fl->ifl_cidx; in iflib_netmap_rxsync()
1214 bus_dmamap_sync(fl->ifl_buf_tag, in iflib_netmap_rxsync()
1215 fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); in iflib_netmap_rxsync()
1217 fl->ifl_cidx = nic_i = nm_next(nic_i, lim); in iflib_netmap_rxsync()
1374 iflib_fl_t fl; in iru_init() local
1376 fl = &rxq->ifr_fl[flid]; in iru_init()
1377 iru->iru_paddrs = fl->ifl_bus_addrs; in iru_init()
1378 iru->iru_idxs = fl->ifl_rxd_idxs; in iru_init()
1380 iru->iru_buf_size = fl->ifl_buf_size; in iru_init()
1381 iru->iru_flidx = fl->ifl_id; in iru_init()
1891 iflib_fl_t fl; in iflib_rxsd_alloc() local
1897 fl = rxq->ifr_fl; in iflib_rxsd_alloc()
1898 for (int i = 0; i < rxq->ifr_nfl; i++, fl++) { in iflib_rxsd_alloc()
1899 fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */ in iflib_rxsd_alloc()
1912 &fl->ifl_buf_tag); in iflib_rxsd_alloc()
1920 if (!(fl->ifl_sds.ifsd_m = in iflib_rxsd_alloc()
1930 if (!(fl->ifl_sds.ifsd_cl = in iflib_rxsd_alloc()
1940 if (!(fl->ifl_sds.ifsd_ba = in iflib_rxsd_alloc()
1952 if (!(fl->ifl_sds.ifsd_map = in iflib_rxsd_alloc()
1960 err = bus_dmamap_create(fl->ifl_buf_tag, 0, in iflib_rxsd_alloc()
1961 &fl->ifl_sds.ifsd_map[i]); in iflib_rxsd_alloc()
2006 iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) in iflib_fl_refill() argument
2018 MPASS(count <= fl->ifl_size - fl->ifl_credits - 1); in iflib_fl_refill()
2020 sd_m = fl->ifl_sds.ifsd_m; in iflib_fl_refill()
2021 sd_map = fl->ifl_sds.ifsd_map; in iflib_fl_refill()
2022 sd_cl = fl->ifl_sds.ifsd_cl; in iflib_fl_refill()
2023 sd_ba = fl->ifl_sds.ifsd_ba; in iflib_fl_refill()
2024 pidx = fl->ifl_pidx; in iflib_fl_refill()
2026 frag_idx = fl->ifl_fragidx; in iflib_fl_refill()
2027 credits = fl->ifl_credits; in iflib_fl_refill()
2032 MPASS(credits + n <= fl->ifl_size); in iflib_fl_refill()
2034 if (pidx < fl->ifl_cidx) in iflib_fl_refill()
2035 MPASS(pidx + n <= fl->ifl_cidx); in iflib_fl_refill()
2036 if (pidx == fl->ifl_cidx && (credits < fl->ifl_size)) in iflib_fl_refill()
2037 MPASS(fl->ifl_gen == 0); in iflib_fl_refill()
2038 if (pidx > fl->ifl_cidx) in iflib_fl_refill()
2039 MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx); in iflib_fl_refill()
2044 iru_init(&iru, fl->ifl_rxq, fl->ifl_id); in iflib_fl_refill()
2053 bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size, in iflib_fl_refill()
2056 bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); in iflib_fl_refill()
2059 cl = uma_zalloc(fl->ifl_zone, M_NOWAIT); in iflib_fl_refill()
2065 err = bus_dmamap_load(fl->ifl_buf_tag, sd_map[frag_idx], in iflib_fl_refill()
2066 cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, in iflib_fl_refill()
2069 uma_zfree(fl->ifl_zone, cl); in iflib_fl_refill()
2076 fl->ifl_cl_enqueued++; in iflib_fl_refill()
2081 bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx], in iflib_fl_refill()
2090 bit_set(fl->ifl_rx_bitmap, frag_idx); in iflib_fl_refill()
2092 fl->ifl_m_enqueued++; in iflib_fl_refill()
2096 fl->ifl_rxd_idxs[i] = frag_idx; in iflib_fl_refill()
2097 fl->ifl_bus_addrs[i] = bus_addr; in iflib_fl_refill()
2100 MPASS(credits <= fl->ifl_size); in iflib_fl_refill()
2101 if (++idx == fl->ifl_size) { in iflib_fl_refill()
2103 fl->ifl_gen = 1; in iflib_fl_refill()
2111 fl->ifl_pidx = idx; in iflib_fl_refill()
2112 fl->ifl_credits = credits; in iflib_fl_refill()
2123 fl->ifl_pidx = idx; in iflib_fl_refill()
2124 fl->ifl_credits = credits; in iflib_fl_refill()
2127 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_fl_refill()
2129 ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, in iflib_fl_refill()
2130 fl->ifl_id, fl->ifl_pidx); in iflib_fl_refill()
2131 if (__predict_true(bit_test(fl->ifl_rx_bitmap, frag_idx))) { in iflib_fl_refill()
2132 fl->ifl_fragidx = frag_idx + 1; in iflib_fl_refill()
2133 if (fl->ifl_fragidx == fl->ifl_size) in iflib_fl_refill()
2134 fl->ifl_fragidx = 0; in iflib_fl_refill()
2136 fl->ifl_fragidx = frag_idx; in iflib_fl_refill()
2144 iflib_fl_refill_all(if_ctx_t ctx, iflib_fl_t fl) in iflib_fl_refill_all() argument
2157 int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1; in iflib_fl_refill_all()
2159 …int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - … in iflib_fl_refill_all()
2162 MPASS(fl->ifl_credits <= fl->ifl_size); in iflib_fl_refill_all()
2166 return (iflib_fl_refill(ctx, fl, reclaimable)); in iflib_fl_refill_all()
2182 iflib_fl_bufs_free(iflib_fl_t fl) in iflib_fl_bufs_free() argument
2184 iflib_dma_info_t idi = fl->ifl_ifdi; in iflib_fl_bufs_free()
2188 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2189 struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; in iflib_fl_bufs_free()
2190 caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; in iflib_fl_bufs_free()
2193 sd_map = fl->ifl_sds.ifsd_map[i]; in iflib_fl_bufs_free()
2194 bus_dmamap_sync(fl->ifl_buf_tag, sd_map, in iflib_fl_bufs_free()
2196 bus_dmamap_unload(fl->ifl_buf_tag, sd_map); in iflib_fl_bufs_free()
2197 uma_zfree(fl->ifl_zone, *sd_cl); in iflib_fl_bufs_free()
2208 fl->ifl_m_dequeued++; in iflib_fl_bufs_free()
2209 fl->ifl_cl_dequeued++; in iflib_fl_bufs_free()
2213 for (i = 0; i < fl->ifl_size; i++) { in iflib_fl_bufs_free()
2214 MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); in iflib_fl_bufs_free()
2215 MPASS(fl->ifl_sds.ifsd_m[i] == NULL); in iflib_fl_bufs_free()
2221 fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0; in iflib_fl_bufs_free()
2231 iflib_fl_setup(iflib_fl_t fl) in iflib_fl_setup() argument
2233 iflib_rxq_t rxq = fl->ifl_rxq; in iflib_fl_setup()
2238 bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1); in iflib_fl_setup()
2242 iflib_fl_bufs_free(fl); in iflib_fl_setup()
2244 MPASS(fl->ifl_credits == 0); in iflib_fl_setup()
2245 qidx = rxq->ifr_fl_offset + fl->ifl_id; in iflib_fl_setup()
2247 fl->ifl_buf_size = scctx->isc_rxd_buf_size[qidx]; in iflib_fl_setup()
2249 fl->ifl_buf_size = ctx->ifc_rx_mbuf_sz; in iflib_fl_setup()
2254 fl->ifl_buf_size = iflib_get_mbuf_size_for(fl->ifl_buf_size); in iflib_fl_setup()
2255 if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size) in iflib_fl_setup()
2256 ctx->ifc_max_fl_buf_size = fl->ifl_buf_size; in iflib_fl_setup()
2257 fl->ifl_cltype = m_gettype(fl->ifl_buf_size); in iflib_fl_setup()
2258 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_fl_setup()
2266 MPASS(fl->ifl_size > 0); in iflib_fl_setup()
2267 (void)iflib_fl_refill(ctx, fl, min(128, fl->ifl_size - 1)); in iflib_fl_setup()
2268 if (min(128, fl->ifl_size - 1) != fl->ifl_credits) in iflib_fl_setup()
2274 MPASS(fl->ifl_ifdi != NULL); in iflib_fl_setup()
2275 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_fl_setup()
2288 iflib_fl_t fl; in iflib_rx_sds_free() local
2293 fl = &rxq->ifr_fl[i]; in iflib_rx_sds_free()
2294 if (fl->ifl_buf_tag != NULL) { in iflib_rx_sds_free()
2295 if (fl->ifl_sds.ifsd_map != NULL) { in iflib_rx_sds_free()
2296 for (j = 0; j < fl->ifl_size; j++) { in iflib_rx_sds_free()
2298 fl->ifl_buf_tag, in iflib_rx_sds_free()
2299 fl->ifl_sds.ifsd_map[j], in iflib_rx_sds_free()
2302 fl->ifl_buf_tag, in iflib_rx_sds_free()
2303 fl->ifl_sds.ifsd_map[j]); in iflib_rx_sds_free()
2305 fl->ifl_buf_tag, in iflib_rx_sds_free()
2306 fl->ifl_sds.ifsd_map[j]); in iflib_rx_sds_free()
2309 bus_dma_tag_destroy(fl->ifl_buf_tag); in iflib_rx_sds_free()
2310 fl->ifl_buf_tag = NULL; in iflib_rx_sds_free()
2312 free(fl->ifl_sds.ifsd_m, M_IFLIB); in iflib_rx_sds_free()
2313 free(fl->ifl_sds.ifsd_cl, M_IFLIB); in iflib_rx_sds_free()
2314 free(fl->ifl_sds.ifsd_ba, M_IFLIB); in iflib_rx_sds_free()
2315 free(fl->ifl_sds.ifsd_map, M_IFLIB); in iflib_rx_sds_free()
2316 free(fl->ifl_rx_bitmap, M_IFLIB); in iflib_rx_sds_free()
2317 fl->ifl_sds.ifsd_m = NULL; in iflib_rx_sds_free()
2318 fl->ifl_sds.ifsd_cl = NULL; in iflib_rx_sds_free()
2319 fl->ifl_sds.ifsd_ba = NULL; in iflib_rx_sds_free()
2320 fl->ifl_sds.ifsd_map = NULL; in iflib_rx_sds_free()
2321 fl->ifl_rx_bitmap = NULL; in iflib_rx_sds_free()
2423 iflib_fl_t fl; in iflib_init_locked() local
2477 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_init_locked()
2478 if (iflib_fl_setup(fl)) { in iflib_init_locked()
2530 iflib_fl_t fl; in iflib_stop() local
2582 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_stop()
2583 iflib_fl_bufs_free(fl); in iflib_stop()
2588 calc_next_rxd(iflib_fl_t fl, int cidx) in calc_next_rxd() argument
2594 nrxd = fl->ifl_size; in calc_next_rxd()
2595 size = fl->ifl_rxd_size; in calc_next_rxd()
2596 start = fl->ifl_ifdi->idi_vaddr; in calc_next_rxd()
2607 prefetch_pkts(iflib_fl_t fl, int cidx) in prefetch_pkts() argument
2610 int nrxd = fl->ifl_size; in prefetch_pkts()
2614 prefetch(&fl->ifl_sds.ifsd_m[nextptr]); in prefetch_pkts()
2615 prefetch(&fl->ifl_sds.ifsd_cl[nextptr]); in prefetch_pkts()
2616 next_rxd = calc_next_rxd(fl, cidx); in prefetch_pkts()
2618 prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd-1)]); in prefetch_pkts()
2619 prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd-1)]); in prefetch_pkts()
2620 prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd-1)]); in prefetch_pkts()
2621 prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd-1)]); in prefetch_pkts()
2622 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd-1)]); in prefetch_pkts()
2623 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd-1)]); in prefetch_pkts()
2624 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd-1)]); in prefetch_pkts()
2625 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd-1)]); in prefetch_pkts()
2633 iflib_fl_t fl; in rxd_frag_to_sd() local
2641 fl = &rxq->ifr_fl[flid]; in rxd_frag_to_sd()
2642 sd->ifsd_fl = fl; in rxd_frag_to_sd()
2643 m = fl->ifl_sds.ifsd_m[cidx]; in rxd_frag_to_sd()
2644 sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; in rxd_frag_to_sd()
2645 fl->ifl_credits--; in rxd_frag_to_sd()
2647 fl->ifl_m_dequeued++; in rxd_frag_to_sd()
2650 prefetch_pkts(fl, cidx); in rxd_frag_to_sd()
2651 next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); in rxd_frag_to_sd()
2652 prefetch(&fl->ifl_sds.ifsd_map[next]); in rxd_frag_to_sd()
2653 map = fl->ifl_sds.ifsd_map[cidx]; in rxd_frag_to_sd()
2655 bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); in rxd_frag_to_sd()
2685 fl->ifl_sds.ifsd_m[cidx] = NULL; in rxd_frag_to_sd()
2691 fl->ifl_sds.ifsd_m[cidx] = NULL; in rxd_frag_to_sd()
2697 bus_dmamap_unload(fl->ifl_buf_tag, map); in rxd_frag_to_sd()
2698 fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); in rxd_frag_to_sd()
2699 if (__predict_false(fl->ifl_cidx == 0)) in rxd_frag_to_sd()
2700 fl->ifl_gen = 0; in rxd_frag_to_sd()
2701 bit_clear(fl->ifl_rx_bitmap, cidx); in rxd_frag_to_sd()
2878 iflib_fl_t fl; in iflib_rxeof() local
2901 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
2902 retval |= iflib_fl_refill_all(ctx, fl); in iflib_rxeof()
2961 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) in iflib_rxeof()
2962 retval |= iflib_fl_refill_all(ctx, fl); in iflib_rxeof()
5592 iflib_fl_t fl = NULL; in iflib_queues_alloc() local
5717 if (!(fl = in iflib_queues_alloc()
5723 rxq->ifr_fl = fl; in iflib_queues_alloc()
5725 fl[j].ifl_rxq = rxq; in iflib_queues_alloc()
5726 fl[j].ifl_id = j; in iflib_queues_alloc()
5727 fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset]; in iflib_queues_alloc()
5728 fl[j].ifl_rxd_size = scctx->isc_rxd_size[j]; in iflib_queues_alloc()
5738 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) in iflib_queues_alloc()
5739 fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB, in iflib_queues_alloc()
6383 iflib_fl_t fl; in iflib_rxd_avail() local
6386 for (i = 0, fl = &rxq->ifr_fl[0]; i < rxq->ifr_nfl; i++, fl++) in iflib_rxd_avail()
6387 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, in iflib_rxd_avail()
6717 iflib_fl_t fl; in iflib_add_device_sysctl_post() local
6829 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { in iflib_add_device_sysctl_post()
6836 &fl->ifl_pidx, 1, "Producer Index"); in iflib_add_device_sysctl_post()
6839 &fl->ifl_cidx, 1, "Consumer Index"); in iflib_add_device_sysctl_post()
6842 &fl->ifl_credits, 1, "credits available"); in iflib_add_device_sysctl_post()
6845 &fl->ifl_buf_size, 1, "buffer size"); in iflib_add_device_sysctl_post()
6849 &fl->ifl_m_enqueued, "mbufs allocated"); in iflib_add_device_sysctl_post()
6852 &fl->ifl_m_dequeued, "mbufs freed"); in iflib_add_device_sysctl_post()
6855 &fl->ifl_cl_enqueued, "clusters allocated"); in iflib_add_device_sysctl_post()
6858 &fl->ifl_cl_dequeued, "clusters freed"); in iflib_add_device_sysctl_post()
6920 iflib_fl_t fl; in iflib_debugnet_event() local
6932 fl = rxq->ifr_fl; in iflib_debugnet_event()
6933 fl->ifl_zone = m_getzone(fl->ifl_buf_size); in iflib_debugnet_event()