118 	union ring_state os, ns;  in drain_ring()  local
144 				ns.state = os.state;  in drain_ring()
145 				ns.cidx = cidx;  in drain_ring()
152 					ns.flags = BUSY;  in drain_ring()
154 					ns.flags = STALLED;  in drain_ring()
156 			    ns.state) == 0);  in drain_ring()
160 			else if (ns.flags == STALLED)  in drain_ring()
174 			ns.state = os.state;  in drain_ring()
175 			ns.cidx = cidx;  in drain_ring()
178 				ns.flags = BUSY;  in drain_ring()
182 				ns.flags = coalescing ? COALESCING : IDLE;  in drain_ring()
186 				ns.flags = TOO_BUSY;  in drain_ring()
192 		} while (atomic_fcmpset_acq_64(&r->state, &os.state, ns.state) == 0);  in drain_ring()
195 			MPASS(ns.flags == BUSY);  in drain_ring()
200 		if (ns.flags == IDLE || ns.flags == COALESCING) {  in drain_ring()
201 			MPASS(ns.pidx_tail == cidx);  in drain_ring()
202 			if (ns.pidx_head != ns.pidx_tail)  in drain_ring()
213 		pidx = ns.pidx_tail;  in drain_ring()
219 		MPASS(ns.flags == BUSY);  in drain_ring()
221 		MPASS(ns.flags == IDLE || ns.flags == COALESCING ||  in drain_ring()
222 		    ns.flags == STALLED);  in drain_ring()
230 	union ring_state ns;  in drain_txpkts()  local
244 		ns.state = os.state;  in drain_txpkts()
247 		if (ns.cidx == ns.pidx_tail)  in drain_txpkts()
248 			ns.flags = IDLE;  in drain_txpkts()
250 			ns.flags = BUSY;  in drain_txpkts()
251 	} while (atomic_fcmpset_acq_64(&r->state, &os.state, ns.state) == 0);  in drain_txpkts()
254 	if (ns.flags == BUSY)  in drain_txpkts()
347 	union ring_state os, ns;  in mp_ring_enqueue()  local
384 		ns.state = os.state;  in mp_ring_enqueue()
385 		ns.pidx_head = increment_idx(r, os.pidx_head, n);  in mp_ring_enqueue()
390 				ns.pidx_tail = increment_idx(r, os.pidx_tail, n);  in mp_ring_enqueue()
393 			ns.flags = BUSY;  in mp_ring_enqueue()
396 			ns.flags = TAKING_OVER;  in mp_ring_enqueue()
399 		if (atomic_fcmpset_64(&r->state, &os.state, ns.state))  in mp_ring_enqueue()
406 	pidx_stop = ns.pidx_head;  in mp_ring_enqueue()
428 	while (ns.pidx_tail != pidx_start) {  in mp_ring_enqueue()
430 		ns.state = atomic_load_64(&r->state);  in mp_ring_enqueue()
448 		ns.state = os.state;  in mp_ring_enqueue()
449 		ns.pidx_tail = pidx_stop;  in mp_ring_enqueue()
454 			ns.flags = BUSY;  in mp_ring_enqueue()
456 	} while (atomic_fcmpset_rel_64(&r->state, &os.state, ns.state) == 0);  in mp_ring_enqueue()
479 	union ring_state os, ns;  in mp_ring_check_drainage()  local
484 		ns.state = os.state;  in mp_ring_check_drainage()
485 		ns.flags = BUSY;  in mp_ring_check_drainage()
486 		if (atomic_cmpset_acq_64(&r->state, os.state, ns.state)) {  in mp_ring_check_drainage()
493 		ns.state = os.state;  in mp_ring_check_drainage()
494 		ns.flags = BUSY;  in mp_ring_check_drainage()
495 		if (atomic_cmpset_acq_64(&r->state, os.state, ns.state)) {  in mp_ring_check_drainage()
497 			drain_txpkts(r, ns, budget);  in mp_ring_check_drainage()