Lines Matching refs:os
94 drain_ring_locked(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget) in drain_ring_locked() argument
98 uint16_t cidx = os.cidx; in drain_ring_locked()
99 uint16_t pidx = os.pidx_tail; in drain_ring_locked()
101 MPASS(os.flags == BUSY); in drain_ring_locked()
113 os.state = ns.state = r->state; in drain_ring_locked()
137 os.state = ns.state = r->state; in drain_ring_locked()
168 drain_ring_lockless(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget) in drain_ring_lockless() argument
172 uint16_t cidx = os.cidx; in drain_ring_lockless()
173 uint16_t pidx = os.pidx_tail; in drain_ring_lockless()
175 MPASS(os.flags == BUSY); in drain_ring_lockless()
188 os.state = r->state; in drain_ring_lockless()
190 ns.state = os.state; in drain_ring_lockless()
193 } while (atomic_fcmpset_64(&r->state, &os.state, in drain_ring_lockless()
216 os.state = r->state; in drain_ring_lockless()
218 ns.state = os.state; in drain_ring_lockless()
221 } while (atomic_fcmpset_acq_64(&r->state, &os.state, in drain_ring_lockless()
321 union ring_state os, ns; in ifmp_ring_enqueue() local
333 os.state = r->state; in ifmp_ring_enqueue()
334 if (n >= space_available(r, os)) { in ifmp_ring_enqueue()
336 MPASS(os.flags != IDLE); in ifmp_ring_enqueue()
338 if (os.flags == STALLED) in ifmp_ring_enqueue()
342 ns.state = os.state; in ifmp_ring_enqueue()
343 ns.pidx_head = increment_idx(r, os.pidx_head, n); in ifmp_ring_enqueue()
345 pidx_start = os.pidx_head; in ifmp_ring_enqueue()
370 os.state = ns.state = r->state; in ifmp_ring_enqueue()
373 if (os.flags == IDLE) in ifmp_ring_enqueue()
385 if (os.flags != BUSY) in ifmp_ring_enqueue()
386 drain_ring_locked(r, ns, os.flags, budget); in ifmp_ring_enqueue()
396 union ring_state os, ns; in ifmp_ring_enqueue() local
407 os.state = r->state; in ifmp_ring_enqueue()
409 if (n >= space_available(r, os)) { in ifmp_ring_enqueue()
411 MPASS(os.flags != IDLE); in ifmp_ring_enqueue()
412 if (os.flags == STALLED) in ifmp_ring_enqueue()
416 ns.state = os.state; in ifmp_ring_enqueue()
417 ns.pidx_head = increment_idx(r, os.pidx_head, n); in ifmp_ring_enqueue()
419 if (atomic_fcmpset_64(&r->state, &os.state, ns.state)) in ifmp_ring_enqueue()
424 pidx_start = os.pidx_head; in ifmp_ring_enqueue()
449 os.state = r->state; in ifmp_ring_enqueue()
451 ns.state = os.state; in ifmp_ring_enqueue()
454 if (os.flags == IDLE) in ifmp_ring_enqueue()
458 } while (atomic_fcmpset_rel_64(&r->state, &os.state, ns.state) == 0); in ifmp_ring_enqueue()
467 if (os.flags != BUSY) in ifmp_ring_enqueue()
468 drain_ring_lockless(r, ns, os.flags, budget); in ifmp_ring_enqueue()
478 union ring_state os, ns; in ifmp_ring_check_drainage() local
480 os.state = r->state; in ifmp_ring_check_drainage()
481 if ((os.flags != STALLED && os.flags != ABDICATED) || // Only continue in STALLED and ABDICATED in ifmp_ring_check_drainage()
482 os.pidx_head != os.pidx_tail || // Require work to be available in ifmp_ring_check_drainage()
483 (os.flags != ABDICATED && r->can_drain(r) == 0)) // Can either drain, or everyone left in ifmp_ring_check_drainage()
486 MPASS(os.cidx != os.pidx_tail); /* implied by STALLED */ in ifmp_ring_check_drainage()
487 ns.state = os.state; in ifmp_ring_check_drainage()
492 if (r->state != os.state) { in ifmp_ring_check_drainage()
497 drain_ring_locked(r, ns, os.flags, budget); in ifmp_ring_check_drainage()
504 if (!atomic_cmpset_acq_64(&r->state, os.state, ns.state)) in ifmp_ring_check_drainage()
507 drain_ring_lockless(r, ns, os.flags, budget); in ifmp_ring_check_drainage()