96 	union ring_state ns;  in drain_ring_locked()  local
113 			os.state = ns.state = r->state;  in drain_ring_locked()
114 			ns.cidx = cidx;  in drain_ring_locked()
115 			ns.flags = STALLED;  in drain_ring_locked()
116 			r->state = ns.state;  in drain_ring_locked()
137 		os.state = ns.state = r->state;  in drain_ring_locked()
138 		ns.cidx = cidx;  in drain_ring_locked()
139 		ns.flags = state_to_flags(ns, total >= budget);  in drain_ring_locked()
140 		r->state = ns.state;  in drain_ring_locked()
142 		if (ns.flags == ABDICATED)  in drain_ring_locked()
144 		if (ns.flags != BUSY) {  in drain_ring_locked()
146 			MPASS(ns.flags != STALLED);  in drain_ring_locked()
158 		pidx = ns.pidx_tail;  in drain_ring_locked()
170 	union ring_state ns;  in drain_ring_lockless()  local
190 				ns.state = os.state;  in drain_ring_lockless()
191 				ns.cidx = cidx;  in drain_ring_lockless()
192 				ns.flags = STALLED;  in drain_ring_lockless()
194 			    ns.state) == 0);  in drain_ring_lockless()
218 			ns.state = os.state;  in drain_ring_lockless()
219 			ns.cidx = cidx;  in drain_ring_lockless()
220 			ns.flags = state_to_flags(ns, total >= budget);  in drain_ring_lockless()
222 		    ns.state) == 0);  in drain_ring_lockless()
225 		if (ns.flags == ABDICATED)  in drain_ring_lockless()
227 		if (ns.flags != BUSY) {  in drain_ring_lockless()
229 			MPASS(ns.flags != STALLED);  in drain_ring_lockless()
241 		pidx = ns.pidx_tail;  in drain_ring_lockless()
321 	union ring_state os, ns;  in ifmp_ring_enqueue()  local
342 	ns.state = os.state;  in ifmp_ring_enqueue()
343 	ns.pidx_head = increment_idx(r, os.pidx_head, n);  in ifmp_ring_enqueue()
344 	r->state = ns.state;  in ifmp_ring_enqueue()
346 	pidx_stop = ns.pidx_head;  in ifmp_ring_enqueue()
353 	while (ns.pidx_tail != pidx_start) {  in ifmp_ring_enqueue()
355 		ns.state = r->state;  in ifmp_ring_enqueue()
370 	os.state = ns.state = r->state;  in ifmp_ring_enqueue()
371 	ns.pidx_tail = pidx_stop;  in ifmp_ring_enqueue()
374 			ns.flags = ABDICATED;  in ifmp_ring_enqueue()
376 		ns.flags = BUSY;  in ifmp_ring_enqueue()
377 	r->state = ns.state;  in ifmp_ring_enqueue()
386 			drain_ring_locked(r, ns, os.flags, budget);  in ifmp_ring_enqueue()
396 	union ring_state os, ns;  in ifmp_ring_enqueue()  local
416 		ns.state = os.state;  in ifmp_ring_enqueue()
417 		ns.pidx_head = increment_idx(r, os.pidx_head, n);  in ifmp_ring_enqueue()
419 		if (atomic_fcmpset_64(&r->state, &os.state, ns.state))  in ifmp_ring_enqueue()
425 	pidx_stop = ns.pidx_head;  in ifmp_ring_enqueue()
432 	while (ns.pidx_tail != pidx_start) {  in ifmp_ring_enqueue()
434 		ns.state = r->state;  in ifmp_ring_enqueue()
451 		ns.state = os.state;  in ifmp_ring_enqueue()
452 		ns.pidx_tail = pidx_stop;  in ifmp_ring_enqueue()
455 				ns.flags = ABDICATED;  in ifmp_ring_enqueue()
457 			ns.flags = BUSY;  in ifmp_ring_enqueue()
458 	} while (atomic_fcmpset_rel_64(&r->state, &os.state, ns.state) == 0);  in ifmp_ring_enqueue()
468 			drain_ring_lockless(r, ns, os.flags, budget);  in ifmp_ring_enqueue()
478 	union ring_state os, ns;  in ifmp_ring_check_drainage()  local
487 	ns.state = os.state;  in ifmp_ring_check_drainage()
488 	ns.flags = BUSY;  in ifmp_ring_check_drainage()
496 	r->state = ns.state;  in ifmp_ring_check_drainage()
497 	drain_ring_locked(r, ns, os.flags, budget);  in ifmp_ring_check_drainage()
504 	if (!atomic_cmpset_acq_64(&r->state, os.state, ns.state))  in ifmp_ring_check_drainage()
507 	drain_ring_lockless(r, ns, os.flags, budget);  in ifmp_ring_check_drainage()