44 static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i)  in verify_update_old_key()  argument
47 	struct bch_fs *c = trans->c;  in verify_update_old_key()
49 	struct bkey_s_c k = bch2_btree_path_peek_slot_exact(trans->paths + i->path, &u);  in verify_update_old_key()
51 	if (unlikely(trans->journal_replay_not_finished)) {  in verify_update_old_key()
67 static inline struct btree_path_level *insert_l(struct btree_trans *trans, struct btree_insert_entr…  in insert_l()  argument
69 	return (trans->paths + i->path)->l + i->level;  in insert_l()
72 static inline bool same_leaf_as_prev(struct btree_trans *trans,  in same_leaf_as_prev()  argument
75 	return i != trans->updates &&  in same_leaf_as_prev()
76 		insert_l(trans, &i[0])->b == insert_l(trans, &i[-1])->b;  in same_leaf_as_prev()
79 static inline bool same_leaf_as_next(struct btree_trans *trans,  in same_leaf_as_next()  argument
82 	return i + 1 < trans->updates + trans->nr_updates &&  in same_leaf_as_next()
83 		insert_l(trans, &i[0])->b == insert_l(trans, &i[1])->b;  in same_leaf_as_next()
86 inline void bch2_btree_node_prep_for_write(struct btree_trans *trans,  in bch2_btree_node_prep_for_write()  argument
90 	struct bch_fs *c = trans->c;  in bch2_btree_node_prep_for_write()
94 		bch2_trans_node_reinit_iter(trans, b);  in bch2_btree_node_prep_for_write()
101 		bch2_btree_init_next(trans, b);  in bch2_btree_node_prep_for_write()
104 static noinline int trans_lock_write_fail(struct btree_trans *trans, struct btree_insert_entry *i)  in trans_lock_write_fail()  argument
106 	while (--i >= trans->updates) {  in trans_lock_write_fail()
107 		if (same_leaf_as_prev(trans, i))  in trans_lock_write_fail()
110 		bch2_btree_node_unlock_write(trans, trans->paths + i->path, insert_l(trans, i)->b);  in trans_lock_write_fail()
113 	trace_and_count(trans->c, trans_restart_would_deadlock_write, trans);  in trans_lock_write_fail()
114 	return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock_write);  in trans_lock_write_fail()
117 static inline int bch2_trans_lock_write(struct btree_trans *trans)  in bch2_trans_lock_write()  argument
119 	EBUG_ON(trans->write_locked);  in bch2_trans_lock_write()
121 	trans_for_each_update(trans, i) {  in bch2_trans_lock_write()
122 		if (same_leaf_as_prev(trans, i))  in bch2_trans_lock_write()
125 		if (bch2_btree_node_lock_write(trans, trans->paths + i->path, &insert_l(trans, i)->b->c))  in bch2_trans_lock_write()
126 			return trans_lock_write_fail(trans, i);  in bch2_trans_lock_write()
129 			bch2_btree_node_prep_for_write(trans, trans->paths + i->path, insert_l(trans, i)->b);  in bch2_trans_lock_write()
132 	trans->write_locked = true;  in bch2_trans_lock_write()
136 static inline void bch2_trans_unlock_updates_write(struct btree_trans *trans)  in bch2_trans_unlock_updates_write()  argument
138 	if (likely(trans->write_locked)) {  in bch2_trans_unlock_updates_write()
139 		trans_for_each_update(trans, i)  in bch2_trans_unlock_updates_write()
140 			if (btree_node_locked_type(trans->paths + i->path, i->level) ==  in bch2_trans_unlock_updates_write()
142 				bch2_btree_node_unlock_write_inlined(trans,  in bch2_trans_unlock_updates_write()
143 						trans->paths + i->path, insert_l(trans, i)->b);  in bch2_trans_unlock_updates_write()
144 		trans->write_locked = false;  in bch2_trans_unlock_updates_write()
151 bool bch2_btree_bset_insert_key(struct btree_trans *trans,  in bch2_btree_bset_insert_key()  argument
194 			bch2_btree_path_fix_key_modified(trans, b, k);  in bch2_btree_bset_insert_key()
212 			bch2_btree_path_fix_key_modified(trans, b, k);  in bch2_btree_bset_insert_key()
222 		bch2_btree_node_iter_fix(trans, path, b, node_iter, k,  in bch2_btree_bset_insert_key()
233 	struct btree_trans *trans = bch2_trans_get(c);  in __btree_node_flush()  local
237 	btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read);  in __btree_node_flush()
253 	btree_node_write_if_need(trans, b, SIX_LOCK_read);  in __btree_node_flush()
256 	bch2_trans_put(trans);  in __btree_node_flush()
288 inline void bch2_btree_insert_key_leaf(struct btree_trans *trans,  in bch2_btree_insert_key_leaf()  argument
293 	struct bch_fs *c = trans->c;  in bch2_btree_insert_key_leaf()
301 	if (unlikely(!bch2_btree_bset_insert_key(trans, path, b,  in bch2_btree_insert_key_leaf()
324 		bch2_trans_node_reinit_iter(trans, b);  in bch2_btree_insert_key_leaf()
331 static inline void btree_insert_entry_checks(struct btree_trans *trans,  in btree_insert_entry_checks()  argument
334 	struct btree_path *path = trans->paths + i->path;  in btree_insert_entry_checks()
344 		test_bit(JOURNAL_replay_done, &trans->c->journal.flags) &&  in btree_insert_entry_checks()
346 		bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot) > 0);  in btree_insert_entry_checks()
349 static __always_inline int bch2_trans_journal_res_get(struct btree_trans *trans,  in bch2_trans_journal_res_get()  argument
352 	return bch2_journal_res_get(&trans->c->journal, &trans->journal_res,  in bch2_trans_journal_res_get()
353 				    trans->journal_u64s, flags, trans);  in bch2_trans_journal_res_get()
358 static noinline void journal_transaction_name(struct btree_trans *trans)  in journal_transaction_name()  argument
360 	struct bch_fs *c = trans->c;  in journal_transaction_name()
363 		bch2_journal_add_entry(j, &trans->journal_res,  in journal_transaction_name()
369 	strncpy(l->d, trans->fn, JSET_ENTRY_LOG_U64s * sizeof(u64));  in journal_transaction_name()
372 static inline int btree_key_can_insert(struct btree_trans *trans,  in btree_key_can_insert()  argument
382 btree_key_can_insert_cached_slowpath(struct btree_trans *trans, unsigned flags,  in btree_key_can_insert_cached_slowpath()  argument
389 	bch2_trans_unlock_updates_write(trans);  in btree_key_can_insert_cached_slowpath()
390 	bch2_trans_unlock(trans);  in btree_key_can_insert_cached_slowpath()
394 		bch_err(trans->c, "error allocating memory for key cache key, btree %s u64s %u",  in btree_key_can_insert_cached_slowpath()
399 	ret =   bch2_trans_relock(trans) ?:  in btree_key_can_insert_cached_slowpath()
400 		bch2_trans_lock_write(trans);  in btree_key_can_insert_cached_slowpath()
408 	trans_for_each_update(trans, i)  in btree_key_can_insert_cached_slowpath()
418 static int btree_key_can_insert_cached(struct btree_trans *trans, unsigned flags,  in btree_key_can_insert_cached()  argument
421 	struct bch_fs *c = trans->c;  in btree_key_can_insert_cached()
446 		return btree_key_can_insert_cached_slowpath(trans, flags, path, new_u64s);  in btree_key_can_insert_cached()
448 	trans_for_each_update(trans, i)  in btree_key_can_insert_cached()
459 static int run_one_mem_trigger(struct btree_trans *trans,  in run_one_mem_trigger()  argument
463 	verify_update_old_key(trans, i);  in run_one_mem_trigger()
474 		return bch2_key_trigger(trans, i->btree_id, i->level,  in run_one_mem_trigger()
478 		return bch2_key_trigger_new(trans, i->btree_id, i->level,  in run_one_mem_trigger()
480 		       bch2_key_trigger_old(trans, i->btree_id, i->level,  in run_one_mem_trigger()
484 static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i)  in run_one_trans_trigger()  argument
486 	verify_update_old_key(trans, i);  in run_one_trans_trigger()
508 		return bch2_key_trigger(trans, i->btree_id, i->level, old, bkey_i_to_s(i->k),  in run_one_trans_trigger()
513 		return bch2_key_trigger_old(trans, i->btree_id, i->level, old, flags) ?: 1;  in run_one_trans_trigger()
516 		return bch2_key_trigger_new(trans, i->btree_id, i->level, bkey_i_to_s(i->k), flags) ?: 1;  in run_one_trans_trigger()
522 static int bch2_trans_commit_run_triggers(struct btree_trans *trans)  in bch2_trans_commit_run_triggers()  argument
526 	while (sort_id_start < trans->nr_updates) {  in bch2_trans_commit_run_triggers()
527 		unsigned i, sort_id = trans->updates[sort_id_start].sort_order;  in bch2_trans_commit_run_triggers()
543 			     i < trans->nr_updates && trans->updates[i].sort_order <= sort_id;  in bch2_trans_commit_run_triggers()
545 				if (trans->updates[i].sort_order < sort_id) {  in bch2_trans_commit_run_triggers()
550 				int ret = run_one_trans_trigger(trans, trans->updates + i);  in bch2_trans_commit_run_triggers()
562 	trans_for_each_update(trans, i)  in bch2_trans_commit_run_triggers()
570 static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans)  in bch2_trans_commit_run_gc_triggers()  argument
572 	trans_for_each_update(trans, i)  in bch2_trans_commit_run_gc_triggers()
574 		    gc_visited(trans->c, gc_pos_btree(i->btree_id, i->level, i->k->k.p))) {  in bch2_trans_commit_run_gc_triggers()
575 			int ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_gc);  in bch2_trans_commit_run_gc_triggers()
584 bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,  in bch2_trans_commit_write_locked()  argument
588 	struct bch_fs *c = trans->c;  in bch2_trans_commit_write_locked()
593 	bch2_trans_verify_not_unlocked_or_in_restart(trans);  in bch2_trans_commit_write_locked()
596 		trace_and_count(c, trans_restart_fault_inject, trans, trace_ip);  in bch2_trans_commit_write_locked()
597 		return btree_trans_restart(trans, BCH_ERR_transaction_restart_fault_inject);  in bch2_trans_commit_write_locked()
606 	prefetch(&trans->c->journal.flags);  in bch2_trans_commit_write_locked()
608 	trans_for_each_update(trans, i) {  in bch2_trans_commit_write_locked()
610 		if (!same_leaf_as_prev(trans, i))  in bch2_trans_commit_write_locked()
615 			? btree_key_can_insert(trans, insert_l(trans, i)->b, u64s)  in bch2_trans_commit_write_locked()
616 			: btree_key_can_insert_cached(trans, flags, trans->paths + i->path, u64s);  in bch2_trans_commit_write_locked()
630 		ret = bch2_trans_journal_res_get(trans,  in bch2_trans_commit_write_locked()
636 		if (unlikely(trans->journal_transaction_names))  in bch2_trans_commit_write_locked()
637 			journal_transaction_name(trans);  in bch2_trans_commit_write_locked()
648 			trans_for_each_update(trans, i)  in bch2_trans_commit_write_locked()
649 				i->k->k.bversion.lo = trans->journal_res.seq;  in bch2_trans_commit_write_locked()
651 			trans_for_each_update(trans, i)  in bch2_trans_commit_write_locked()
655 	h = trans->hooks;  in bch2_trans_commit_write_locked()
657 		ret = h->fn(trans, h);  in bch2_trans_commit_write_locked()
663 	struct jset_entry *entry = trans->journal_entries;  in bch2_trans_commit_write_locked()
666 	for (entry = trans->journal_entries;  in bch2_trans_commit_write_locked()
667 	     entry != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);  in bch2_trans_commit_write_locked()
671 			ret = bch2_accounting_trans_commit_hook(trans, bkey_i_to_accounting(entry->start), flags);  in bch2_trans_commit_write_locked()
678 	bch2_trans_account_disk_usage_change(trans);  in bch2_trans_commit_write_locked()
680 	trans_for_each_update(trans, i)  in bch2_trans_commit_write_locked()
682 			ret = run_one_mem_trigger(trans, i, BTREE_TRIGGER_atomic|i->flags);  in bch2_trans_commit_write_locked()
688 		ret = bch2_trans_commit_run_gc_triggers(trans);  in bch2_trans_commit_write_locked()
698 	for (struct jset_entry *i = trans->journal_entries;  in bch2_trans_commit_write_locked()
699 	     i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);  in bch2_trans_commit_write_locked()
705 			bch2_trans_inconsistent(trans, "invalid journal entry on insert from %s\n",  in bch2_trans_commit_write_locked()
706 						trans->fn);  in bch2_trans_commit_write_locked()
711 	trans_for_each_update(trans, i) {  in bch2_trans_commit_write_locked()
717 			bch2_trans_inconsistent(trans, "invalid bkey on insert from %s -> %ps\n",  in bch2_trans_commit_write_locked()
718 						trans->fn, (void *) i->ip_allocated);  in bch2_trans_commit_write_locked()
721 		btree_insert_entry_checks(trans, i);  in bch2_trans_commit_write_locked()
728 		trans_for_each_update(trans, i) {  in bch2_trans_commit_write_locked()
735 			verify_update_old_key(trans, i);  in bch2_trans_commit_write_locked()
737 			if (trans->journal_transaction_names) {  in bch2_trans_commit_write_locked()
738 				entry = bch2_journal_add_entry(j, &trans->journal_res,  in bch2_trans_commit_write_locked()
746 			entry = bch2_journal_add_entry(j, &trans->journal_res,  in bch2_trans_commit_write_locked()
753 		memcpy_u64s_small(journal_res_entry(&c->journal, &trans->journal_res),  in bch2_trans_commit_write_locked()
754 				  trans->journal_entries,  in bch2_trans_commit_write_locked()
755 				  trans->journal_entries_u64s);  in bch2_trans_commit_write_locked()
757 		trans->journal_res.offset	+= trans->journal_entries_u64s;  in bch2_trans_commit_write_locked()
758 		trans->journal_res.u64s		-= trans->journal_entries_u64s;  in bch2_trans_commit_write_locked()
760 		if (trans->journal_seq)  in bch2_trans_commit_write_locked()
761 			*trans->journal_seq = trans->journal_res.seq;  in bch2_trans_commit_write_locked()
764 	trans_for_each_update(trans, i) {  in bch2_trans_commit_write_locked()
765 		struct btree_path *path = trans->paths + i->path;  in bch2_trans_commit_write_locked()
768 			bch2_btree_insert_key_leaf(trans, path, i->k, trans->journal_res.seq);  in bch2_trans_commit_write_locked()
770 			bch2_btree_insert_key_cached(trans, flags, i);  in bch2_trans_commit_write_locked()
772 			bch2_btree_key_cache_drop(trans, path);  in bch2_trans_commit_write_locked()
780 	for (struct jset_entry *entry2 = trans->journal_entries;  in bch2_trans_commit_write_locked()
785 			bch2_accounting_trans_commit_revert(trans,  in bch2_trans_commit_write_locked()
791 static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans)  in bch2_drop_overwrites_from_journal()  argument
799 	trans_for_each_update(trans, i)  in bch2_drop_overwrites_from_journal()
801 			bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p);  in bch2_drop_overwrites_from_journal()
813 static inline int do_bch2_trans_commit(struct btree_trans *trans, unsigned flags,  in do_bch2_trans_commit()  argument
817 	struct bch_fs *c = trans->c;  in do_bch2_trans_commit()
820 	for (unsigned idx = 0; idx < trans->nr_updates; idx++) {  in do_bch2_trans_commit()
821 		struct btree_insert_entry *i = trans->updates + idx;  in do_bch2_trans_commit()
828 		if (!same_leaf_as_next(trans, i)) {  in do_bch2_trans_commit()
830 				ret = bch2_foreground_maybe_merge(trans, i->path,  in do_bch2_trans_commit()
840 	ret = bch2_trans_lock_write(trans);  in do_bch2_trans_commit()
844 	ret = bch2_trans_commit_write_locked(trans, flags, stopped_at, trace_ip);  in do_bch2_trans_commit()
846 	if (!ret && unlikely(trans->journal_replay_not_finished))  in do_bch2_trans_commit()
847 		bch2_drop_overwrites_from_journal(trans);  in do_bch2_trans_commit()
849 	bch2_trans_unlock_updates_write(trans);  in do_bch2_trans_commit()
851 	if (!ret && trans->journal_pin)  in do_bch2_trans_commit()
852 		bch2_journal_pin_add(&c->journal, trans->journal_res.seq,  in do_bch2_trans_commit()
853 				     trans->journal_pin,  in do_bch2_trans_commit()
861 		bch2_journal_res_put(&c->journal, &trans->journal_res);  in do_bch2_trans_commit()
877 int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags,  in bch2_trans_commit_error()  argument
881 	struct bch_fs *c = trans->c;  in bch2_trans_commit_error()
895 		ret = drop_locks_do(trans,  in bch2_trans_commit_error()
896 			bch2_trans_journal_res_get(trans,  in bch2_trans_commit_error()
904 		ret = bch2_btree_split_leaf(trans, i->path, flags);  in bch2_trans_commit_error()
906 			trace_and_count(c, trans_restart_btree_node_split, trans,  in bch2_trans_commit_error()
907 					trace_ip, trans->paths + i->path);  in bch2_trans_commit_error()
910 		ret = drop_locks_do(trans,  in bch2_trans_commit_error()
911 			bch2_accounting_update_sb(trans));  in bch2_trans_commit_error()
914 		bch2_trans_unlock(trans);  in bch2_trans_commit_error()
916 		trace_and_count(c, trans_blocked_journal_reclaim, trans, trace_ip);  in bch2_trans_commit_error()
927 		ret = bch2_trans_relock(trans);  in bch2_trans_commit_error()
934 	BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted);  in bch2_trans_commit_error()
949 do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans)  in do_bch2_trans_commit_to_journal_replay()  argument
951 	struct bch_fs *c = trans->c;  in do_bch2_trans_commit_to_journal_replay()
955 	trans_for_each_update(trans, i) {  in do_bch2_trans_commit_to_journal_replay()
961 	for (struct jset_entry *i = trans->journal_entries;  in do_bch2_trans_commit_to_journal_replay()
962 	     i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);  in do_bch2_trans_commit_to_journal_replay()
974 int __bch2_trans_commit(struct btree_trans *trans, unsigned flags)  in __bch2_trans_commit()  argument
977 	struct bch_fs *c = trans->c;  in __bch2_trans_commit()
980 	bch2_trans_verify_not_unlocked_or_in_restart(trans);  in __bch2_trans_commit()
982 	ret = trans_maybe_inject_restart(trans, _RET_IP_);  in __bch2_trans_commit()
986 	if (!trans->nr_updates &&  in __bch2_trans_commit()
987 	    !trans->journal_entries_u64s)  in __bch2_trans_commit()
990 	ret = bch2_trans_commit_run_triggers(trans);  in __bch2_trans_commit()
997 			ret = do_bch2_trans_commit_to_journal_replay(trans);  in __bch2_trans_commit()
1005 	trans->journal_u64s		= trans->journal_entries_u64s;  in __bch2_trans_commit()
1006 	trans->journal_transaction_names = READ_ONCE(c->opts.journal_transaction_names);  in __bch2_trans_commit()
1007 	if (trans->journal_transaction_names)  in __bch2_trans_commit()
1008 		trans->journal_u64s += jset_u64s(JSET_ENTRY_LOG_U64s);  in __bch2_trans_commit()
1010 	trans_for_each_update(trans, i) {  in __bch2_trans_commit()
1011 		struct btree_path *path = trans->paths + i->path;  in __bch2_trans_commit()
1015 		ret = bch2_btree_path_upgrade(trans, path, i->level + 1);  in __bch2_trans_commit()
1028 		trans->journal_u64s += jset_u64s(i->k->k.u64s);  in __bch2_trans_commit()
1031 		if (trans->journal_transaction_names)  in __bch2_trans_commit()
1032 			trans->journal_u64s += jset_u64s(i->old_k.u64s);  in __bch2_trans_commit()
1035 	if (trans->extra_disk_res) {  in __bch2_trans_commit()
1036 		ret = bch2_disk_reservation_add(c, trans->disk_res,  in __bch2_trans_commit()
1037 				trans->extra_disk_res,  in __bch2_trans_commit()
1045 	bch2_trans_verify_not_unlocked_or_in_restart(trans);  in __bch2_trans_commit()
1047 		memset(&trans->journal_res, 0, sizeof(trans->journal_res));  in __bch2_trans_commit()
1048 	memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta));  in __bch2_trans_commit()
1050 	ret = do_bch2_trans_commit(trans, flags, &errored_at, _RET_IP_);  in __bch2_trans_commit()
1053 	bch2_trans_verify_locks(trans);  in __bch2_trans_commit()
1058 	trace_and_count(c, transaction_commit, trans, _RET_IP_);  in __bch2_trans_commit()
1064 		bch2_trans_downgrade(trans);  in __bch2_trans_commit()
1065 	bch2_trans_reset_updates(trans);  in __bch2_trans_commit()
1069 	ret = bch2_trans_commit_error(trans, flags, errored_at, ret, _RET_IP_);  in __bch2_trans_commit()