104 linuxkpi_napi_schedule_prep(struct napi_struct *napi)  in linuxkpi_napi_schedule_prep()  argument
108 	NAPI_TRACE(napi);  in linuxkpi_napi_schedule_prep()
112 		old = READ_ONCE(napi->state);  in linuxkpi_napi_schedule_prep()
116 			NAPI_TRACE(napi);  in linuxkpi_napi_schedule_prep()
126 	} while (atomic_cmpset_acq_long(&napi->state, old, new) == 0);  in linuxkpi_napi_schedule_prep()
128 	NAPI_TRACE(napi);  in linuxkpi_napi_schedule_prep()
133 lkpi___napi_schedule_dd(struct napi_struct *napi)  in lkpi___napi_schedule_dd()  argument
140 	NAPI_TRACE2D(napi, rc);  in lkpi___napi_schedule_dd()
141 	if (napi->poll != NULL)  in lkpi___napi_schedule_dd()
142 		rc = napi->poll(napi, napi->budget);  in lkpi___napi_schedule_dd()
143 	napi->rx_count += rc;  in lkpi___napi_schedule_dd()
147 	if (rc >= napi->budget)  in lkpi___napi_schedule_dd()
151 	if (test_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &napi->state))  in lkpi___napi_schedule_dd()
155 		new = old = READ_ONCE(napi->state);  in lkpi___napi_schedule_dd()
158 	} while (atomic_cmpset_acq_long(&napi->state, old, new) == 0);  in lkpi___napi_schedule_dd()
160 	NAPI_TRACE2D(napi, rc);  in lkpi___napi_schedule_dd()
164 linuxkpi___napi_schedule(struct napi_struct *napi)  in linuxkpi___napi_schedule()  argument
168 	NAPI_TRACE(napi);  in linuxkpi___napi_schedule()
169 	if (test_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state)) {  in linuxkpi___napi_schedule()
170 		clear_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &napi->state);  in linuxkpi___napi_schedule()
171 		clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state);  in linuxkpi___napi_schedule()
172 		NAPI_TRACE(napi);  in linuxkpi___napi_schedule()
177 		lkpi___napi_schedule_dd(napi);  in linuxkpi___napi_schedule()
179 		rc = taskqueue_enqueue(napi->dev->napi_tq, &napi->napi_task);  in linuxkpi___napi_schedule()
180 		NAPI_TRACE2D(napi, rc);  in linuxkpi___napi_schedule()
189 linuxkpi_napi_schedule(struct napi_struct *napi)  in linuxkpi_napi_schedule()  argument
192 	NAPI_TRACE(napi);  in linuxkpi_napi_schedule()
198 	if (napi_schedule_prep(napi)) {  in linuxkpi_napi_schedule()
199 		__napi_schedule(napi);  in linuxkpi_napi_schedule()
207 linuxkpi_napi_reschedule(struct napi_struct *napi)  in linuxkpi_napi_reschedule()  argument
210 	NAPI_TRACE(napi);  in linuxkpi_napi_reschedule()
213 	if (napi_schedule_prep(napi))  in linuxkpi_napi_reschedule()
214 		__napi_schedule(napi);  in linuxkpi_napi_reschedule()
218 linuxkpi_napi_complete_done(struct napi_struct *napi, int ret)  in linuxkpi_napi_complete_done()  argument
222 	NAPI_TRACE(napi);  in linuxkpi_napi_complete_done()
227 		new = old = READ_ONCE(napi->state);  in linuxkpi_napi_complete_done()
236 	} while (atomic_cmpset_acq_long(&napi->state, old, new) == 0);  in linuxkpi_napi_complete_done()
238 	NAPI_TRACE(napi);  in linuxkpi_napi_complete_done()
242 		__napi_schedule(napi);  in linuxkpi_napi_complete_done()
250 linuxkpi_napi_complete(struct napi_struct *napi)  in linuxkpi_napi_complete()  argument
253 	NAPI_TRACE(napi);  in linuxkpi_napi_complete()
254 	return (napi_complete_done(napi, 0));  in linuxkpi_napi_complete()
258 linuxkpi_napi_disable(struct napi_struct *napi)  in linuxkpi_napi_disable()  argument
260 	NAPI_TRACE(napi);  in linuxkpi_napi_disable()
261 	set_bit(LKPI_NAPI_FLAG_DISABLE_PENDING, &napi->state);  in linuxkpi_napi_disable()
262 	while (test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state))  in linuxkpi_napi_disable()
264 	clear_bit(LKPI_NAPI_FLAG_DISABLE_PENDING, &napi->state);  in linuxkpi_napi_disable()
268 linuxkpi_napi_enable(struct napi_struct *napi)  in linuxkpi_napi_enable()  argument
271 	NAPI_TRACE(napi);  in linuxkpi_napi_enable()
272 	KASSERT(!test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state),  in linuxkpi_napi_enable()
273 	    ("%s: enabling napi %p already scheduled\n", __func__, napi));  in linuxkpi_napi_enable()
276 	clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state);  in linuxkpi_napi_enable()
280 linuxkpi_napi_synchronize(struct napi_struct *napi)  in linuxkpi_napi_synchronize()  argument
282 	NAPI_TRACE(napi);  in linuxkpi_napi_synchronize()
285 	while (test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state))  in linuxkpi_napi_synchronize()
297 	struct napi_struct *napi;  in lkpi_napi_task()  local
302 	napi = ctx;  in lkpi_napi_task()
303 	KASSERT(napi->poll != NULL, ("%s: napi %p poll is NULL\n",  in lkpi_napi_task()
304 	    __func__, napi));  in lkpi_napi_task()
306 	NAPI_TRACE_TASK(napi, pending, napi->budget);  in lkpi_napi_task()
307 	count = napi->poll(napi, napi->budget);  in lkpi_napi_task()
308 	napi->rx_count += count;  in lkpi_napi_task()
309 	NAPI_TRACE_TASK(napi, pending, count);  in lkpi_napi_task()
316 	if (count >= napi->budget) {  in lkpi_napi_task()
323 		__napi_schedule(napi);  in lkpi_napi_task()
330 linuxkpi_netif_napi_add(struct net_device *ndev, struct napi_struct *napi,  in linuxkpi_netif_napi_add()  argument
334 	napi->dev = ndev;  in linuxkpi_netif_napi_add()
335 	napi->poll = napi_poll;  in linuxkpi_netif_napi_add()
336 	napi->budget = NAPI_POLL_WEIGHT;  in linuxkpi_netif_napi_add()
338 	INIT_LIST_HEAD(&napi->rx_list);  in linuxkpi_netif_napi_add()
339 	napi->rx_count = 0;  in linuxkpi_netif_napi_add()
341 	TASK_INIT(&napi->napi_task, 0, lkpi_napi_task, napi);  in linuxkpi_netif_napi_add()
344 	TAILQ_INSERT_TAIL(&ndev->napi_head, napi, entry);  in linuxkpi_netif_napi_add()
348 	clear_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state);  in linuxkpi_netif_napi_add()
352 lkpi_netif_napi_del_locked(struct napi_struct *napi)  in lkpi_netif_napi_del_locked()  argument
356 	ndev = napi->dev;  in lkpi_netif_napi_del_locked()
359 	set_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state);  in lkpi_netif_napi_del_locked()
360 	TAILQ_REMOVE(&ndev->napi_head, napi, entry);  in lkpi_netif_napi_del_locked()
361 	while (taskqueue_cancel(ndev->napi_tq, &napi->napi_task, NULL) != 0)  in lkpi_netif_napi_del_locked()
362 		taskqueue_drain(ndev->napi_tq, &napi->napi_task);  in lkpi_netif_napi_del_locked()
366 linuxkpi_netif_napi_del(struct napi_struct *napi)  in linuxkpi_netif_napi_del()  argument
370 	ndev = napi->dev;  in linuxkpi_netif_napi_del()
372 	lkpi_netif_napi_del_locked(napi);  in linuxkpi_netif_napi_del()
422 	struct napi_struct *napi, *temp;  in linuxkpi_free_netdev()  local
425 	TAILQ_FOREACH_SAFE(napi, &ndev->napi_head, entry, temp) {  in linuxkpi_free_netdev()
426 		lkpi_netif_napi_del_locked(napi);  in linuxkpi_free_netdev()