Lines Matching refs:kring
225 struct netmap_kring *kring = NULL; in generic_netmap_unregister() local
240 for_each_rx_kring(r, kring, na) { in generic_netmap_unregister()
244 mbq_safe_purge(&kring->rx_queue); in generic_netmap_unregister()
252 for_each_tx_kring(r, kring, na) { in generic_netmap_unregister()
257 mtx_lock_spin(&kring->tx_event_lock); in generic_netmap_unregister()
258 if (kring->tx_event) { in generic_netmap_unregister()
259 SET_MBUF_DESTRUCTOR(kring->tx_event, NULL); in generic_netmap_unregister()
261 kring->tx_event = NULL; in generic_netmap_unregister()
262 mtx_unlock_spin(&kring->tx_event_lock); in generic_netmap_unregister()
268 for_each_rx_kring(r, kring, na) { in generic_netmap_unregister()
269 mbq_safe_fini(&kring->rx_queue); in generic_netmap_unregister()
272 for_each_tx_kring(r, kring, na) { in generic_netmap_unregister()
273 callout_drain(&kring->tx_event_callout); in generic_netmap_unregister()
274 mtx_destroy(&kring->tx_event_lock); in generic_netmap_unregister()
275 if (kring->tx_pool == NULL) { in generic_netmap_unregister()
280 if (kring->tx_pool[i]) { in generic_netmap_unregister()
281 m_freem(kring->tx_pool[i]); in generic_netmap_unregister()
284 nm_os_free(kring->tx_pool); in generic_netmap_unregister()
285 kring->tx_pool = NULL; in generic_netmap_unregister()
305 struct netmap_kring *kring = NULL; in generic_netmap_register() local
331 for_each_rx_kring(r, kring, na) { in generic_netmap_register()
338 mbq_safe_init(&kring->rx_queue); in generic_netmap_register()
346 for_each_tx_kring(r, kring, na) { in generic_netmap_register()
347 kring->tx_pool = NULL; in generic_netmap_register()
349 for_each_tx_kring(r, kring, na) { in generic_netmap_register()
350 kring->tx_pool = in generic_netmap_register()
352 if (!kring->tx_pool) { in generic_netmap_register()
357 mtx_init(&kring->tx_event_lock, "tx_event_lock", in generic_netmap_register()
359 callout_init_mtx(&kring->tx_event_callout, in generic_netmap_register()
360 &kring->tx_event_lock, in generic_netmap_register()
367 for_each_tx_kring(r, kring, na) { in generic_netmap_register()
370 kring->tx_pool[i] = NULL; in generic_netmap_register()
373 kring->tx_event = NULL; in generic_netmap_register()
412 for_each_tx_kring(r, kring, na) { in generic_netmap_register()
413 mtx_destroy(&kring->tx_event_lock); in generic_netmap_register()
414 if (kring->tx_pool == NULL) { in generic_netmap_register()
417 nm_os_free(kring->tx_pool); in generic_netmap_register()
418 kring->tx_pool = NULL; in generic_netmap_register()
420 for_each_rx_kring(r, kring, na) { in generic_netmap_register()
421 mbq_safe_fini(&kring->rx_queue); in generic_netmap_register()
438 struct netmap_kring *kring; in generic_mbuf_dtor() local
459 kring = na->tx_rings[r]; in generic_mbuf_dtor()
460 mtx_lock_spin(&kring->tx_event_lock); in generic_mbuf_dtor()
461 if (kring->tx_event == m) { in generic_mbuf_dtor()
462 kring->tx_event = NULL; in generic_mbuf_dtor()
465 mtx_unlock_spin(&kring->tx_event_lock); in generic_mbuf_dtor()
502 generic_netmap_tx_clean(struct netmap_kring *kring, int txqdisc) in generic_netmap_tx_clean() argument
504 u_int const lim = kring->nkr_num_slots - 1; in generic_netmap_tx_clean()
505 u_int nm_i = nm_next(kring->nr_hwtail, lim); in generic_netmap_tx_clean()
506 u_int hwcur = kring->nr_hwcur; in generic_netmap_tx_clean()
508 struct mbuf **tx_pool = kring->tx_pool; in generic_netmap_tx_clean()
510 nm_prdis("hwcur = %d, hwtail = %d", kring->nr_hwcur, kring->nr_hwtail); in generic_netmap_tx_clean()
537 mtx_lock_spin(&kring->tx_event_lock); in generic_netmap_tx_clean()
538 event_consumed = (kring->tx_event == NULL); in generic_netmap_tx_clean()
539 mtx_unlock_spin(&kring->tx_event_lock); in generic_netmap_tx_clean()
556 kring->nr_hwtail = nm_prev(nm_i, lim); in generic_netmap_tx_clean()
557 nm_prdis("tx completed [%d] -> hwtail %d", n, kring->nr_hwtail); in generic_netmap_tx_clean()
590 struct netmap_kring *kring = arg; in generic_tx_callout() local
592 kring->tx_event = NULL; in generic_tx_callout()
593 mtx_unlock_spin(&kring->tx_event_lock); in generic_tx_callout()
594 netmap_generic_irq(kring->na, kring->ring_id, NULL); in generic_tx_callout()
599 generic_set_tx_event(struct netmap_kring *kring, u_int hwcur) in generic_set_tx_event() argument
601 u_int lim = kring->nkr_num_slots - 1; in generic_set_tx_event()
604 u_int ntc = nm_next(kring->nr_hwtail, lim); /* next to clean */ in generic_set_tx_event()
628 m = kring->tx_pool[e]; in generic_set_tx_event()
634 mtx_lock_spin(&kring->tx_event_lock); in generic_set_tx_event()
635 if (kring->tx_event) { in generic_set_tx_event()
637 mtx_unlock_spin(&kring->tx_event_lock); in generic_set_tx_event()
642 kring->tx_event = m; in generic_set_tx_event()
656 callout_reset_sbt_curcpu(&kring->tx_event_callout, SBT_1MS, 0, in generic_set_tx_event()
657 generic_tx_callout, kring, 0); in generic_set_tx_event()
659 mtx_unlock_spin(&kring->tx_event_lock); in generic_set_tx_event()
661 kring->tx_pool[e] = NULL; in generic_set_tx_event()
678 generic_netmap_txsync(struct netmap_kring *kring, int flags) in generic_netmap_txsync() argument
680 struct netmap_adapter *na = kring->na; in generic_netmap_txsync()
683 struct netmap_ring *ring = kring->ring; in generic_netmap_txsync()
685 u_int const lim = kring->nkr_num_slots - 1; in generic_netmap_txsync()
686 u_int const head = kring->rhead; in generic_netmap_txsync()
687 u_int ring_nr = kring->ring_id; in generic_netmap_txsync()
696 nm_i = kring->nr_hwcur; in generic_netmap_txsync()
706 if (gna->txqdisc && nm_kr_txempty(kring)) { in generic_netmap_txsync()
713 kring->nr_hwtail); in generic_netmap_txsync()
732 m = kring->tx_pool[nm_i]; in generic_netmap_txsync()
734 kring->tx_pool[nm_i] = m = in generic_netmap_txsync()
779 generic_set_tx_event(kring, nm_i); in generic_netmap_txsync()
780 if (generic_netmap_tx_clean(kring, gna->txqdisc)) { in generic_netmap_txsync()
813 kring->nr_hwcur = nm_i; in generic_netmap_txsync()
820 if (!gna->txqdisc && (flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring))) { in generic_netmap_txsync()
826 generic_set_tx_event(kring, nm_i); in generic_netmap_txsync()
832 generic_netmap_tx_clean(kring, gna->txqdisc); in generic_netmap_txsync()
852 struct netmap_kring *kring; in generic_rx_handler() local
860 kring = na->rx_rings[r]; in generic_rx_handler()
862 if (kring->nr_mode == NKR_NETMAP_OFF) { in generic_rx_handler()
876 } else if (unlikely(mbq_len(&kring->rx_queue) > na->num_rx_desc)) { in generic_rx_handler()
880 mbq_safe_enqueue(&kring->rx_queue, m); in generic_rx_handler()
910 generic_netmap_rxsync(struct netmap_kring *kring, int flags) in generic_netmap_rxsync() argument
912 struct netmap_ring *ring = kring->ring; in generic_netmap_rxsync()
913 struct netmap_adapter *na = kring->na; in generic_netmap_rxsync()
916 u_int const lim = kring->nkr_num_slots - 1; in generic_netmap_rxsync()
917 u_int const head = kring->rhead; in generic_netmap_rxsync()
918 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; in generic_netmap_rxsync()
929 return netmap_ring_reinit(kring); in generic_netmap_rxsync()
937 nm_i = kring->nr_hwcur; in generic_netmap_rxsync()
946 kring->nr_hwcur = head; in generic_netmap_rxsync()
956 nm_i = kring->nr_hwtail; /* First empty slot in the receive ring. */ in generic_netmap_rxsync()
962 avail = nm_prev(kring->nr_hwcur, lim) - nm_i; in generic_netmap_rxsync()
974 mbq_lock(&kring->rx_queue); in generic_netmap_rxsync()
976 m = mbq_peek(&kring->rx_queue); in generic_netmap_rxsync()
988 mbq_dequeue(&kring->rx_queue); in generic_netmap_rxsync()
1005 mbq_unlock(&kring->rx_queue); in generic_netmap_rxsync()
1009 nm_i = kring->nr_hwtail; in generic_netmap_rxsync()
1028 return netmap_ring_reinit(kring); in generic_netmap_rxsync()
1044 kring->nr_hwtail = nm_i; in generic_netmap_rxsync()
1047 kring->nr_kflags &= ~NKR_PENDINTR; in generic_netmap_rxsync()