63 buf_ring_enqueue(struct buf_ring *br, void *buf)  in buf_ring_enqueue()  argument
74 	for (i = br->br_cons_head; i != br->br_prod_head;  in buf_ring_enqueue()
75 	     i = ((i + 1) & br->br_cons_mask))  in buf_ring_enqueue()
76 		if(br->br_ring[i] == buf)  in buf_ring_enqueue()
78 			    buf, i, br->br_prod_tail, br->br_cons_tail);  in buf_ring_enqueue()
82 		prod_head = br->br_prod_head;  in buf_ring_enqueue()
83 		prod_next = (prod_head + 1) & br->br_prod_mask;  in buf_ring_enqueue()
84 		cons_tail = br->br_cons_tail;  in buf_ring_enqueue()
88 			if (prod_head == br->br_prod_head &&  in buf_ring_enqueue()
89 			    cons_tail == br->br_cons_tail) {  in buf_ring_enqueue()
90 				br->br_drops++;  in buf_ring_enqueue()
96 	} while (!atomic_cmpset_acq_int(&br->br_prod_head, prod_head, prod_next));  in buf_ring_enqueue()
98 	if (br->br_ring[prod_head] != NULL)  in buf_ring_enqueue()
101 	br->br_ring[prod_head] = buf;  in buf_ring_enqueue()
108 	while (br->br_prod_tail != prod_head)  in buf_ring_enqueue()
110 	atomic_store_rel_int(&br->br_prod_tail, prod_next);  in buf_ring_enqueue()
120 buf_ring_dequeue_mc(struct buf_ring *br)  in buf_ring_dequeue_mc()  argument
127 		cons_head = br->br_cons_head;  in buf_ring_dequeue_mc()
128 		cons_next = (cons_head + 1) & br->br_cons_mask;  in buf_ring_dequeue_mc()
130 		if (cons_head == br->br_prod_tail) {  in buf_ring_dequeue_mc()
134 	} while (!atomic_cmpset_acq_int(&br->br_cons_head, cons_head, cons_next));  in buf_ring_dequeue_mc()
136 	buf = br->br_ring[cons_head];  in buf_ring_dequeue_mc()
138 	br->br_ring[cons_head] = NULL;  in buf_ring_dequeue_mc()
145 	while (br->br_cons_tail != cons_head)  in buf_ring_dequeue_mc()
148 	atomic_store_rel_int(&br->br_cons_tail, cons_next);  in buf_ring_dequeue_mc()
160 buf_ring_dequeue_sc(struct buf_ring *br)  in buf_ring_dequeue_sc()  argument
195 	cons_head = atomic_load_acq_32(&br->br_cons_head);  in buf_ring_dequeue_sc()
197 	cons_head = br->br_cons_head;  in buf_ring_dequeue_sc()
199 	prod_tail = atomic_load_acq_32(&br->br_prod_tail);  in buf_ring_dequeue_sc()
201 	cons_next = (cons_head + 1) & br->br_cons_mask;  in buf_ring_dequeue_sc()
203 	cons_next_next = (cons_head + 2) & br->br_cons_mask;  in buf_ring_dequeue_sc()
211 		prefetch(br->br_ring[cons_next]);  in buf_ring_dequeue_sc()
213 			prefetch(br->br_ring[cons_next_next]);  in buf_ring_dequeue_sc()
216 	br->br_cons_head = cons_next;  in buf_ring_dequeue_sc()
217 	buf = br->br_ring[cons_head];  in buf_ring_dequeue_sc()
220 	br->br_ring[cons_head] = NULL;  in buf_ring_dequeue_sc()
221 	if (!mtx_owned(br->br_lock))  in buf_ring_dequeue_sc()
223 	if (br->br_cons_tail != cons_head)  in buf_ring_dequeue_sc()
225 		    br->br_cons_tail, cons_head);  in buf_ring_dequeue_sc()
227 	br->br_cons_tail = cons_next;  in buf_ring_dequeue_sc()
237 buf_ring_advance_sc(struct buf_ring *br)  in buf_ring_advance_sc()  argument
242 	cons_head = br->br_cons_head;  in buf_ring_advance_sc()
243 	prod_tail = br->br_prod_tail;  in buf_ring_advance_sc()
245 	cons_next = (cons_head + 1) & br->br_cons_mask;  in buf_ring_advance_sc()
248 	br->br_cons_head = cons_next;  in buf_ring_advance_sc()
250 	br->br_ring[cons_head] = NULL;  in buf_ring_advance_sc()
252 	br->br_cons_tail = cons_next;  in buf_ring_advance_sc()
272 buf_ring_putback_sc(struct buf_ring *br, void *new)  in buf_ring_putback_sc()  argument
274 	KASSERT(br->br_cons_head != br->br_prod_tail,   in buf_ring_putback_sc()
276 	br->br_ring[br->br_cons_head] = new;  in buf_ring_putback_sc()
285 buf_ring_peek(struct buf_ring *br)  in buf_ring_peek()  argument
289 	if ((br->br_lock != NULL) && !mtx_owned(br->br_lock))  in buf_ring_peek()
298 	if (br->br_cons_head == br->br_prod_tail)  in buf_ring_peek()
301 	return (br->br_ring[br->br_cons_head]);  in buf_ring_peek()
305 buf_ring_peek_clear_sc(struct buf_ring *br)  in buf_ring_peek_clear_sc()  argument
310 	if (!mtx_owned(br->br_lock))  in buf_ring_peek_clear_sc()
314 	if (br->br_cons_head == br->br_prod_tail)  in buf_ring_peek_clear_sc()
336 	ret = br->br_ring[br->br_cons_head];  in buf_ring_peek_clear_sc()
337 	br->br_ring[br->br_cons_head] = NULL;  in buf_ring_peek_clear_sc()
340 	return (br->br_ring[br->br_cons_head]);  in buf_ring_peek_clear_sc()
345 buf_ring_full(struct buf_ring *br)  in buf_ring_full()  argument
348 	return (((br->br_prod_head + 1) & br->br_prod_mask) == br->br_cons_tail);  in buf_ring_full()
352 buf_ring_empty(struct buf_ring *br)  in buf_ring_empty()  argument
355 	return (br->br_cons_head == br->br_prod_tail);  in buf_ring_empty()
359 buf_ring_count(struct buf_ring *br)  in buf_ring_count()  argument
362 	return ((br->br_prod_size + br->br_prod_tail - br->br_cons_tail)  in buf_ring_count()
363 	    & br->br_prod_mask);  in buf_ring_count()
368 void buf_ring_free(struct buf_ring *br, struct malloc_type *type);