117 					  struct btree_write_buffered_key *wb)  in wb_flush_one_slowpath()  argument
123 	trans->journal_res.seq = wb->journal_seq;  in wb_flush_one_slowpath()
125 	return bch2_trans_update(trans, iter, &wb->k,  in wb_flush_one_slowpath()
135 			       struct btree_write_buffered_key *wb,  in wb_flush_one()  argument
143 	EBUG_ON(!wb->journal_seq);  in wb_flush_one()
145 	EBUG_ON(trans->c->btree_write_buffer.flushing.pin.seq > wb->journal_seq);  in wb_flush_one()
151 	if (!*accounting_accumulated && wb->k.k.type == KEY_TYPE_accounting) {  in wb_flush_one()
156 			bch2_accounting_accumulate(bkey_i_to_accounting(&wb->k),  in wb_flush_one()
179 	if (unlikely(!bch2_btree_node_insert_fits(path->l[0].b, wb->k.k.u64s))) {  in wb_flush_one()
181 		return wb_flush_one_slowpath(trans, iter, wb);  in wb_flush_one()
184 	bch2_btree_insert_key_leaf(trans, path, &wb->k, wb->journal_seq);  in wb_flush_one()
201 			  struct btree_write_buffered_key *wb)  in btree_write_buffered_insert()  argument
206 	bch2_trans_iter_init(trans, &iter, wb->btree, bkey_start_pos(&wb->k.k),  in btree_write_buffered_insert()
209 	trans->journal_res.seq = wb->journal_seq;  in btree_write_buffered_insert()
212 		bch2_trans_update(trans, &iter, &wb->k,  in btree_write_buffered_insert()
218 static void move_keys_from_inc_to_flushing(struct btree_write_buffer *wb)  in move_keys_from_inc_to_flushing()  argument
220 	struct bch_fs *c = container_of(wb, struct bch_fs, btree_write_buffer);  in move_keys_from_inc_to_flushing()
223 	if (!wb->inc.keys.nr)  in move_keys_from_inc_to_flushing()
226 	bch2_journal_pin_add(j, wb->inc.keys.data[0].journal_seq, &wb->flushing.pin,  in move_keys_from_inc_to_flushing()
229 	darray_resize(&wb->flushing.keys, min_t(size_t, 1U << 20, wb->flushing.keys.nr + wb->inc.keys.nr));  in move_keys_from_inc_to_flushing()
230 	darray_resize(&wb->sorted, wb->flushing.keys.size);  in move_keys_from_inc_to_flushing()
232 	if (!wb->flushing.keys.nr && wb->sorted.size >= wb->inc.keys.nr) {  in move_keys_from_inc_to_flushing()
233 		swap(wb->flushing.keys, wb->inc.keys);  in move_keys_from_inc_to_flushing()
237 	size_t nr = min(darray_room(wb->flushing.keys),  in move_keys_from_inc_to_flushing()
238 			wb->sorted.size - wb->flushing.keys.nr);  in move_keys_from_inc_to_flushing()
239 	nr = min(nr, wb->inc.keys.nr);  in move_keys_from_inc_to_flushing()
241 	memcpy(&darray_top(wb->flushing.keys),  in move_keys_from_inc_to_flushing()
242 	       wb->inc.keys.data,  in move_keys_from_inc_to_flushing()
243 	       sizeof(wb->inc.keys.data[0]) * nr);  in move_keys_from_inc_to_flushing()
245 	memmove(wb->inc.keys.data,  in move_keys_from_inc_to_flushing()
246 		wb->inc.keys.data + nr,  in move_keys_from_inc_to_flushing()
247 	       sizeof(wb->inc.keys.data[0]) * (wb->inc.keys.nr - nr));  in move_keys_from_inc_to_flushing()
249 	wb->flushing.keys.nr	+= nr;  in move_keys_from_inc_to_flushing()
250 	wb->inc.keys.nr		-= nr;  in move_keys_from_inc_to_flushing()
252 	if (!wb->inc.keys.nr)  in move_keys_from_inc_to_flushing()
253 		bch2_journal_pin_drop(j, &wb->inc.pin);  in move_keys_from_inc_to_flushing()
255 		bch2_journal_pin_update(j, wb->inc.keys.data[0].journal_seq, &wb->inc.pin,  in move_keys_from_inc_to_flushing()
264 	BUG_ON(wb->sorted.size < wb->flushing.keys.nr);  in move_keys_from_inc_to_flushing()
287 	struct btree_write_buffer *wb = &c->btree_write_buffer;  in bch2_btree_write_buffer_flush_locked()  local
301 	mutex_lock(&wb->inc.lock);  in bch2_btree_write_buffer_flush_locked()
302 	move_keys_from_inc_to_flushing(wb);  in bch2_btree_write_buffer_flush_locked()
303 	mutex_unlock(&wb->inc.lock);  in bch2_btree_write_buffer_flush_locked()
305 	for (size_t i = 0; i < wb->flushing.keys.nr; i++) {  in bch2_btree_write_buffer_flush_locked()
306 		wb->sorted.data[i].idx = i;  in bch2_btree_write_buffer_flush_locked()
307 		wb->sorted.data[i].btree = wb->flushing.keys.data[i].btree;  in bch2_btree_write_buffer_flush_locked()
308 		memcpy(&wb->sorted.data[i].pos, &wb->flushing.keys.data[i].k.k.p, sizeof(struct bpos));  in bch2_btree_write_buffer_flush_locked()
310 	wb->sorted.nr = wb->flushing.keys.nr;  in bch2_btree_write_buffer_flush_locked()
326 	wb_sort(wb->sorted.data, wb->sorted.nr);  in bch2_btree_write_buffer_flush_locked()
328 	darray_for_each(wb->sorted, i) {  in bch2_btree_write_buffer_flush_locked()
329 		struct btree_write_buffered_key *k = &wb->flushing.keys.data[i->idx];  in bch2_btree_write_buffer_flush_locked()
336 		for (struct wb_key_ref *n = i + 1; n < min(i + 4, &darray_top(wb->sorted)); n++)  in bch2_btree_write_buffer_flush_locked()
337 			prefetch(&wb->flushing.keys.data[n->idx]);  in bch2_btree_write_buffer_flush_locked()
347 		if (i + 1 < &darray_top(wb->sorted) &&  in bch2_btree_write_buffer_flush_locked()
349 			struct btree_write_buffered_key *n = &wb->flushing.keys.data[i[1].idx];  in bch2_btree_write_buffer_flush_locked()
429 		trace_and_count(c, write_buffer_flush_slowpath, trans, slowpath, wb->flushing.keys.nr);  in bch2_btree_write_buffer_flush_locked()
431 		sort_nonatomic(wb->flushing.keys.data,  in bch2_btree_write_buffer_flush_locked()
432 			       wb->flushing.keys.nr,  in bch2_btree_write_buffer_flush_locked()
433 			       sizeof(wb->flushing.keys.data[0]),  in bch2_btree_write_buffer_flush_locked()
436 		darray_for_each(wb->flushing.keys, i) {  in bch2_btree_write_buffer_flush_locked()
447 				bch2_journal_pin_update(j, i->journal_seq, &wb->flushing.pin,  in bch2_btree_write_buffer_flush_locked()
484 			struct btree_write_buffered_key *dst = wb->flushing.keys.data;  in bch2_btree_write_buffer_flush_locked()
486 			darray_for_each(wb->flushing.keys, i)  in bch2_btree_write_buffer_flush_locked()
489 			wb->flushing.keys.nr = dst - wb->flushing.keys.data;  in bch2_btree_write_buffer_flush_locked()
494 		bch2_journal_pin_drop(j, &wb->flushing.pin);  in bch2_btree_write_buffer_flush_locked()
495 		wb->flushing.keys.nr = 0;  in bch2_btree_write_buffer_flush_locked()
499 	trace_write_buffer_flush(trans, wb->flushing.keys.nr, overwritten, fast, 0);  in bch2_btree_write_buffer_flush_locked()
555 	struct btree_write_buffer *wb = &c->btree_write_buffer;  in btree_write_buffer_flush_seq()  local
563 		*did_work |= wb->inc.keys.nr || wb->flushing.keys.nr;  in btree_write_buffer_flush_seq()
569 		mutex_lock(&wb->flushing.lock);  in btree_write_buffer_flush_seq()
571 		mutex_unlock(&wb->flushing.lock);  in btree_write_buffer_flush_seq()
574 		  (wb->inc.pin.seq && wb->inc.pin.seq <= max_seq) ||  in btree_write_buffer_flush_seq()
575 		  (wb->flushing.pin.seq && wb->flushing.pin.seq <= max_seq)));  in btree_write_buffer_flush_seq()
617 	struct btree_write_buffer *wb = &c->btree_write_buffer;  in bch2_btree_write_buffer_flush_nocheck_rw()  local
620 	if (mutex_trylock(&wb->flushing.lock)) {  in bch2_btree_write_buffer_flush_nocheck_rw()
622 		mutex_unlock(&wb->flushing.lock);  in bch2_btree_write_buffer_flush_nocheck_rw()
686 	struct btree_write_buffer *wb = &c->btree_write_buffer;  in bch2_btree_write_buffer_flush_work()  local
689 	mutex_lock(&wb->flushing.lock);  in bch2_btree_write_buffer_flush_work()
693 	mutex_unlock(&wb->flushing.lock);  in bch2_btree_write_buffer_flush_work()
698 static void wb_accounting_sort(struct btree_write_buffer *wb)  in wb_accounting_sort()  argument
700 	eytzinger0_sort(wb->accounting.data, wb->accounting.nr,  in wb_accounting_sort()
701 			sizeof(wb->accounting.data[0]),  in wb_accounting_sort()
708 	struct btree_write_buffer *wb = &c->btree_write_buffer;  in bch2_accounting_key_to_wb_slowpath()  local
713 	int ret = darray_push(&wb->accounting, new);  in bch2_accounting_key_to_wb_slowpath()
717 	wb_accounting_sort(wb);  in bch2_accounting_key_to_wb_slowpath()
725 	struct btree_write_buffer *wb = &c->btree_write_buffer;  in bch2_journal_key_to_wb_slowpath()  local
728 	ret = darray_make_room_gfp(&dst->wb->keys, 1, GFP_KERNEL);  in bch2_journal_key_to_wb_slowpath()
729 	if (!ret && dst->wb == &wb->flushing)  in bch2_journal_key_to_wb_slowpath()
730 		ret = darray_resize(&wb->sorted, wb->flushing.keys.size);  in bch2_journal_key_to_wb_slowpath()
733 		if (dst->wb == &c->btree_write_buffer.flushing) {  in bch2_journal_key_to_wb_slowpath()
734 			mutex_unlock(&dst->wb->lock);  in bch2_journal_key_to_wb_slowpath()
735 			dst->wb = &c->btree_write_buffer.inc;  in bch2_journal_key_to_wb_slowpath()
736 			bch2_journal_pin_add(&c->journal, dst->seq, &dst->wb->pin,  in bch2_journal_key_to_wb_slowpath()
744 	dst->room = darray_room(dst->wb->keys);  in bch2_journal_key_to_wb_slowpath()
745 	if (dst->wb == &wb->flushing)  in bch2_journal_key_to_wb_slowpath()
746 		dst->room = min(dst->room, wb->sorted.size - wb->flushing.keys.nr);  in bch2_journal_key_to_wb_slowpath()
750 	struct btree_write_buffered_key *wb_k = &darray_top(dst->wb->keys);  in bch2_journal_key_to_wb_slowpath()
754 	dst->wb->keys.nr++;  in bch2_journal_key_to_wb_slowpath()
761 	struct btree_write_buffer *wb = &c->btree_write_buffer;  in bch2_journal_keys_to_write_buffer_start()  local
763 	if (mutex_trylock(&wb->flushing.lock)) {  in bch2_journal_keys_to_write_buffer_start()
764 		mutex_lock(&wb->inc.lock);  in bch2_journal_keys_to_write_buffer_start()
765 		move_keys_from_inc_to_flushing(wb);  in bch2_journal_keys_to_write_buffer_start()
772 		if (!wb->inc.keys.nr) {  in bch2_journal_keys_to_write_buffer_start()
773 			dst->wb = &wb->flushing;  in bch2_journal_keys_to_write_buffer_start()
775 			mutex_unlock(&wb->flushing.lock);  in bch2_journal_keys_to_write_buffer_start()
776 			dst->wb = &wb->inc;  in bch2_journal_keys_to_write_buffer_start()
779 		mutex_lock(&wb->inc.lock);  in bch2_journal_keys_to_write_buffer_start()
780 		dst->wb = &wb->inc;  in bch2_journal_keys_to_write_buffer_start()
783 	dst->room = darray_room(dst->wb->keys);  in bch2_journal_keys_to_write_buffer_start()
784 	if (dst->wb == &wb->flushing)  in bch2_journal_keys_to_write_buffer_start()
785 		dst->room = min(dst->room, wb->sorted.size - wb->flushing.keys.nr);  in bch2_journal_keys_to_write_buffer_start()
788 	bch2_journal_pin_add(&c->journal, seq, &dst->wb->pin,  in bch2_journal_keys_to_write_buffer_start()
791 	darray_for_each(wb->accounting, i)  in bch2_journal_keys_to_write_buffer_start()
797 	struct btree_write_buffer *wb = &c->btree_write_buffer;  in bch2_journal_keys_to_write_buffer_end()  local
801 	darray_for_each(wb->accounting, i)  in bch2_journal_keys_to_write_buffer_end()
810 	if (live_accounting_keys * 2 < wb->accounting.nr) {  in bch2_journal_keys_to_write_buffer_end()
811 		struct btree_write_buffered_key *dst = wb->accounting.data;  in bch2_journal_keys_to_write_buffer_end()
813 		darray_for_each(wb->accounting, src)  in bch2_journal_keys_to_write_buffer_end()
816 		wb->accounting.nr = dst - wb->accounting.data;  in bch2_journal_keys_to_write_buffer_end()
817 		wb_accounting_sort(wb);  in bch2_journal_keys_to_write_buffer_end()
820 	if (!dst->wb->keys.nr)  in bch2_journal_keys_to_write_buffer_end()
821 		bch2_journal_pin_drop(&c->journal, &dst->wb->pin);  in bch2_journal_keys_to_write_buffer_end()
828 	if (dst->wb == &wb->flushing)  in bch2_journal_keys_to_write_buffer_end()
829 		mutex_unlock(&wb->flushing.lock);  in bch2_journal_keys_to_write_buffer_end()
830 	mutex_unlock(&wb->inc.lock);  in bch2_journal_keys_to_write_buffer_end()
835 static int wb_keys_resize(struct btree_write_buffer_keys *wb, size_t new_size)  in wb_keys_resize()  argument
837 	if (wb->keys.size >= new_size)  in wb_keys_resize()
840 	if (!mutex_trylock(&wb->lock))  in wb_keys_resize()
843 	int ret = darray_resize(&wb->keys, new_size);  in wb_keys_resize()
844 	mutex_unlock(&wb->lock);  in wb_keys_resize()
850 	struct btree_write_buffer *wb = &c->btree_write_buffer;  in bch2_btree_write_buffer_resize()  local
852 	return wb_keys_resize(&wb->flushing, new_size) ?:  in bch2_btree_write_buffer_resize()
853 		wb_keys_resize(&wb->inc, new_size);  in bch2_btree_write_buffer_resize()
858 	struct btree_write_buffer *wb = &c->btree_write_buffer;  in bch2_fs_btree_write_buffer_exit()  local
860 	BUG_ON((wb->inc.keys.nr || wb->flushing.keys.nr) &&  in bch2_fs_btree_write_buffer_exit()
863 	darray_exit(&wb->accounting);  in bch2_fs_btree_write_buffer_exit()
864 	darray_exit(&wb->sorted);  in bch2_fs_btree_write_buffer_exit()
865 	darray_exit(&wb->flushing.keys);  in bch2_fs_btree_write_buffer_exit()
866 	darray_exit(&wb->inc.keys);  in bch2_fs_btree_write_buffer_exit()
871 	struct btree_write_buffer *wb = &c->btree_write_buffer;  in bch2_fs_btree_write_buffer_init()  local
873 	mutex_init(&wb->inc.lock);  in bch2_fs_btree_write_buffer_init()
874 	mutex_init(&wb->flushing.lock);  in bch2_fs_btree_write_buffer_init()
875 	INIT_WORK(&wb->flush_work, bch2_btree_write_buffer_flush_work);  in bch2_fs_btree_write_buffer_init()
880 	return  darray_make_room(&wb->inc.keys, initial_size) ?:  in bch2_fs_btree_write_buffer_init()
881 		darray_make_room(&wb->flushing.keys, initial_size) ?:  in bch2_fs_btree_write_buffer_init()
882 		darray_make_room(&wb->sorted, initial_size);  in bch2_fs_btree_write_buffer_init()