113 linuxkpi_napi_schedule_prep(struct napi_struct *napi)  in linuxkpi_napi_schedule_prep()  argument
117 	NAPI_TRACE(napi);  in linuxkpi_napi_schedule_prep()
121 		old = READ_ONCE(napi->_flags);  in linuxkpi_napi_schedule_prep()
125 			NAPI_TRACE(napi);  in linuxkpi_napi_schedule_prep()
135 	} while (atomic_cmpset_acq_long(&napi->_flags, old, new) == 0);  in linuxkpi_napi_schedule_prep()
137 	NAPI_TRACE(napi);  in linuxkpi_napi_schedule_prep()
142 lkpi___napi_schedule_dd(struct napi_struct *napi)  in lkpi___napi_schedule_dd()  argument
149 	NAPI_TRACE2D(napi, rc);  in lkpi___napi_schedule_dd()
150 	if (napi->poll != NULL)  in lkpi___napi_schedule_dd()
151 		rc = napi->poll(napi, napi->budget);  in lkpi___napi_schedule_dd()
152 	napi->rx_count += rc;  in lkpi___napi_schedule_dd()
156 	if (rc >= napi->budget)  in lkpi___napi_schedule_dd()
160 	if (test_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &napi->_flags))  in lkpi___napi_schedule_dd()
164 		new = old = READ_ONCE(napi->_flags);  in lkpi___napi_schedule_dd()
167 	} while (atomic_cmpset_acq_long(&napi->_flags, old, new) == 0);  in lkpi___napi_schedule_dd()
169 	NAPI_TRACE2D(napi, rc);  in lkpi___napi_schedule_dd()
173 linuxkpi___napi_schedule(struct napi_struct *napi)  in linuxkpi___napi_schedule()  argument
177 	NAPI_TRACE(napi);  in linuxkpi___napi_schedule()
178 	if (test_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->_flags)) {  in linuxkpi___napi_schedule()
179 		clear_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &napi->_flags);  in linuxkpi___napi_schedule()
180 		clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->_flags);  in linuxkpi___napi_schedule()
181 		NAPI_TRACE(napi);  in linuxkpi___napi_schedule()
186 		lkpi___napi_schedule_dd(napi);  in linuxkpi___napi_schedule()
188 		rc = taskqueue_enqueue(napi->dev->napi_tq, &napi->napi_task);  in linuxkpi___napi_schedule()
189 		NAPI_TRACE2D(napi, rc);  in linuxkpi___napi_schedule()
198 linuxkpi_napi_schedule(struct napi_struct *napi)  in linuxkpi_napi_schedule()  argument
201 	NAPI_TRACE(napi);  in linuxkpi_napi_schedule()
207 	if (napi_schedule_prep(napi))  in linuxkpi_napi_schedule()
208 		__napi_schedule(napi);  in linuxkpi_napi_schedule()
212 linuxkpi_napi_reschedule(struct napi_struct *napi)  in linuxkpi_napi_reschedule()  argument
215 	NAPI_TRACE(napi);  in linuxkpi_napi_reschedule()
218 	if (napi_schedule_prep(napi))  in linuxkpi_napi_reschedule()
219 		__napi_schedule(napi);  in linuxkpi_napi_reschedule()
223 linuxkpi_napi_complete_done(struct napi_struct *napi, int ret)  in linuxkpi_napi_complete_done()  argument
227 	NAPI_TRACE(napi);  in linuxkpi_napi_complete_done()
232 		new = old = READ_ONCE(napi->_flags);  in linuxkpi_napi_complete_done()
241 	} while (atomic_cmpset_acq_long(&napi->_flags, old, new) == 0);  in linuxkpi_napi_complete_done()
243 	NAPI_TRACE(napi);  in linuxkpi_napi_complete_done()
247 		__napi_schedule(napi);  in linuxkpi_napi_complete_done()
255 linuxkpi_napi_complete(struct napi_struct *napi)  in linuxkpi_napi_complete()  argument
258 	NAPI_TRACE(napi);  in linuxkpi_napi_complete()
259 	return (napi_complete_done(napi, 0));  in linuxkpi_napi_complete()
263 linuxkpi_napi_disable(struct napi_struct *napi)  in linuxkpi_napi_disable()  argument
265 	NAPI_TRACE(napi);  in linuxkpi_napi_disable()
266 	set_bit(LKPI_NAPI_FLAG_DISABLE_PENDING, &napi->_flags);  in linuxkpi_napi_disable()
267 	while (test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->_flags))  in linuxkpi_napi_disable()
269 	clear_bit(LKPI_NAPI_FLAG_DISABLE_PENDING, &napi->_flags);  in linuxkpi_napi_disable()
273 linuxkpi_napi_enable(struct napi_struct *napi)  in linuxkpi_napi_enable()  argument
276 	NAPI_TRACE(napi);  in linuxkpi_napi_enable()
277 	KASSERT(!test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->_flags),  in linuxkpi_napi_enable()
278 	    ("%s: enabling napi %p already scheduled\n", __func__, napi));  in linuxkpi_napi_enable()
281 	clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->_flags);  in linuxkpi_napi_enable()
285 linuxkpi_napi_synchronize(struct napi_struct *napi)  in linuxkpi_napi_synchronize()  argument
287 	NAPI_TRACE(napi);  in linuxkpi_napi_synchronize()
290 	while (test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->_flags))  in linuxkpi_napi_synchronize()
302 	struct napi_struct *napi;  in lkpi_napi_task()  local
307 	napi = ctx;  in lkpi_napi_task()
308 	KASSERT(napi->poll != NULL, ("%s: napi %p poll is NULL\n",  in lkpi_napi_task()
309 	    __func__, napi));  in lkpi_napi_task()
311 	NAPI_TRACE_TASK(napi, pending, napi->budget);  in lkpi_napi_task()
312 	count = napi->poll(napi, napi->budget);  in lkpi_napi_task()
313 	napi->rx_count += count;  in lkpi_napi_task()
314 	NAPI_TRACE_TASK(napi, pending, count);  in lkpi_napi_task()
321 	if (count >= napi->budget) {  in lkpi_napi_task()
328 		__napi_schedule(napi);  in lkpi_napi_task()
335 linuxkpi_netif_napi_add(struct net_device *ndev, struct napi_struct *napi,  in linuxkpi_netif_napi_add()  argument
339 	napi->dev = ndev;  in linuxkpi_netif_napi_add()
340 	napi->poll = napi_poll;  in linuxkpi_netif_napi_add()
341 	napi->budget = budget;  in linuxkpi_netif_napi_add()
343 	INIT_LIST_HEAD(&napi->rx_list);  in linuxkpi_netif_napi_add()
344 	napi->rx_count = 0;  in linuxkpi_netif_napi_add()
346 	TASK_INIT(&napi->napi_task, 0, lkpi_napi_task, napi);  in linuxkpi_netif_napi_add()
349 	TAILQ_INSERT_TAIL(&ndev->napi_head, napi, entry);  in linuxkpi_netif_napi_add()
353 	clear_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->_flags);  in linuxkpi_netif_napi_add()
357 lkpi_netif_napi_del_locked(struct napi_struct *napi)  in lkpi_netif_napi_del_locked()  argument
361 	ndev = napi->dev;  in lkpi_netif_napi_del_locked()
364 	set_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->_flags);  in lkpi_netif_napi_del_locked()
365 	TAILQ_REMOVE(&ndev->napi_head, napi, entry);  in lkpi_netif_napi_del_locked()
366 	while (taskqueue_cancel(ndev->napi_tq, &napi->napi_task, NULL) != 0)  in lkpi_netif_napi_del_locked()
367 		taskqueue_drain(ndev->napi_tq, &napi->napi_task);  in lkpi_netif_napi_del_locked()
371 linuxkpi_netif_napi_del(struct napi_struct *napi)  in linuxkpi_netif_napi_del()  argument
375 	ndev = napi->dev;  in linuxkpi_netif_napi_del()
377 	lkpi_netif_napi_del_locked(napi);  in linuxkpi_netif_napi_del()
426 	struct napi_struct *napi, *temp;  in linuxkpi_free_netdev()  local
429 	TAILQ_FOREACH_SAFE(napi, &ndev->napi_head, entry, temp) {  in linuxkpi_free_netdev()
430 		lkpi_netif_napi_del_locked(napi);  in linuxkpi_free_netdev()