Lines Matching refs:os
118 union ring_state os, ns; in drain_ring() local
126 os.state = atomic_load_acq_64(&r->state); in drain_ring()
127 MPASS(os.flags == BUSY); in drain_ring()
129 cidx = os.cidx; in drain_ring()
130 pidx = os.pidx_tail; in drain_ring()
142 os.state = atomic_load_64(&r->state); in drain_ring()
144 ns.state = os.state; in drain_ring()
147 MPASS(os.flags == BUSY || in drain_ring()
148 os.flags == TOO_BUSY || in drain_ring()
149 os.flags == TAKING_OVER); in drain_ring()
151 if (os.flags == TAKING_OVER) in drain_ring()
155 } while (atomic_fcmpset_64(&r->state, &os.state, in drain_ring()
158 if (os.flags == TAKING_OVER) in drain_ring()
169 os.state = atomic_load_64(&r->state); in drain_ring()
171 MPASS(os.flags == BUSY || os.flags == TOO_BUSY || in drain_ring()
172 os.flags == TAKING_OVER); in drain_ring()
174 ns.state = os.state; in drain_ring()
176 if (__predict_false(os.flags == TAKING_OVER)) { in drain_ring()
181 if (cidx == os.pidx_tail) { in drain_ring()
189 MPASS(os.flags == BUSY); in drain_ring()
192 } while (atomic_fcmpset_acq_64(&r->state, &os.state, ns.state) == 0); in drain_ring()
194 if (__predict_false(os.flags == TAKING_OVER)) { in drain_ring()
218 if (os.flags == TAKING_OVER) in drain_ring()
228 drain_txpkts(struct mp_ring *r, union ring_state os, int budget) in drain_txpkts() argument
231 uint16_t cidx = os.cidx; in drain_txpkts()
232 uint16_t pidx = os.pidx_tail; in drain_txpkts()
236 MPASS(os.flags == BUSY); in drain_txpkts()
242 os.state = atomic_load_64(&r->state); in drain_txpkts()
244 ns.state = os.state; in drain_txpkts()
245 MPASS(os.flags == BUSY); in drain_txpkts()
246 MPASS(os.cidx == cidx); in drain_txpkts()
251 } while (atomic_fcmpset_acq_64(&r->state, &os.state, ns.state) == 0); in drain_txpkts()
347 union ring_state os, ns; in mp_ring_enqueue() local
360 os.state = atomic_load_64(&r->state); in mp_ring_enqueue()
363 if (__predict_true(space_available(r, os) >= n)) in mp_ring_enqueue()
368 MPASS(os.flags != IDLE); in mp_ring_enqueue()
369 MPASS(os.flags != COALESCING); in mp_ring_enqueue()
374 if (os.flags == STALLED) in mp_ring_enqueue()
378 os.state = atomic_load_64(&r->state); in mp_ring_enqueue()
384 ns.state = os.state; in mp_ring_enqueue()
385 ns.pidx_head = increment_idx(r, os.pidx_head, n); in mp_ring_enqueue()
386 if (os.flags == IDLE || os.flags == COALESCING) { in mp_ring_enqueue()
387 MPASS(os.pidx_tail == os.cidx); in mp_ring_enqueue()
388 if (os.pidx_head == os.pidx_tail) { in mp_ring_enqueue()
390 ns.pidx_tail = increment_idx(r, os.pidx_tail, n); in mp_ring_enqueue()
394 } else if (os.flags == TOO_BUSY) { in mp_ring_enqueue()
399 if (atomic_fcmpset_64(&r->state, &os.state, ns.state)) in mp_ring_enqueue()
405 pidx_start = os.pidx_head; in mp_ring_enqueue()
445 os.state = atomic_load_64(&r->state); in mp_ring_enqueue()
448 ns.state = os.state; in mp_ring_enqueue()
450 if (os.flags == IDLE || os.flags == COALESCING || in mp_ring_enqueue()
451 (os.flags == STALLED && r->can_drain(r))) { in mp_ring_enqueue()
456 } while (atomic_fcmpset_rel_64(&r->state, &os.state, ns.state) == 0); in mp_ring_enqueue()
479 union ring_state os, ns; in mp_ring_check_drainage() local
481 os.state = atomic_load_64(&r->state); in mp_ring_check_drainage()
482 if (os.flags == STALLED && r->can_drain(r)) { in mp_ring_check_drainage()
483 MPASS(os.cidx != os.pidx_tail); /* implied by STALLED */ in mp_ring_check_drainage()
484 ns.state = os.state; in mp_ring_check_drainage()
486 if (atomic_cmpset_acq_64(&r->state, os.state, ns.state)) { in mp_ring_check_drainage()
491 } else if (os.flags == COALESCING) { in mp_ring_check_drainage()
492 MPASS(os.cidx == os.pidx_tail); in mp_ring_check_drainage()
493 ns.state = os.state; in mp_ring_check_drainage()
495 if (atomic_cmpset_acq_64(&r->state, os.state, ns.state)) { in mp_ring_check_drainage()