// SPDX-License-Identifier: GPL-2.0

#include "bcachefs.h"

#include "alloc/accounting.h"
#include "alloc/buckets.h"
#include "alloc/foreground.h"
#include "alloc/replicas.h"

#include "btree/check.h"
#include "btree/iter.h"
#include "btree/journal_overlay.h"
#include "btree/interior.h"
#include "btree/key_cache.h"
#include "btree/sort.h"
#include "btree/write.h"
#include "btree/write_buffer.h"

#include "journal/journal.h"
#include "journal/read.h"
#include "journal/reclaim.h"
#include "journal/validate.h"

#include "init/error.h"
#include "init/fs.h"

#include "sb/counters.h"

#include "snapshots/snapshot.h"

#include "util/enumerated_ref.h"

#include <linux/prefetch.h>
#include <linux/string_helpers.h>

static const char * const trans_commit_flags_strs[] = {
#define x(n, ...) #n,
	BCH_TRANS_COMMIT_FLAGS()
#undef x
	NULL
};

__cold void bch2_trans_commit_flags_to_text(struct printbuf *out, enum bch_trans_commit_flags flags)
{
	enum bch_watermark watermark = flags & BCH_WATERMARK_MASK;

	prt_printf(out, "watermark=%s", bch2_watermarks[watermark]);

	flags >>= BCH_WATERMARK_BITS;
	if (flags) {
		prt_char(out, ' ');
		bch2_prt_bitflags(out, trans_commit_flags_strs, flags);
	}
}

static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i)
{
#ifdef CONFIG_BCACHEFS_DEBUG
	if (i->key_cache_flushing)
		return;

	struct bch_fs *c = trans->c;
	struct bkey u;
	struct bkey_s_c k = bch2_btree_path_peek_slot_exact(trans->paths + i->path, &u);

	if (unlikely(trans->journal_replay_not_finished)) {
		const struct bkey_i *j_k =
			bch2_journal_keys_peek_slot(c, i->btree_id, i->level, i->k->k.p);

		if (j_k)
			k = bkey_i_to_s_c(j_k);
	}

	/* when updating btree ptrs, mem_ptr may change underneath us, unlocked */
	if (!bkey_fields_eq(k.k, &i->old_k) || k.v != i->old_v) {
		CLASS(printbuf, buf)();
		prt_str(&buf, "updated cached old key doesn't match\n");
		prt_str(&buf, "cached: ");
		bch2_bkey_to_text(&buf, &i->old_k);
		prt_printf(&buf, " %px\n", i->old_v);

		prt_str(&buf, "real:   ");
		bch2_bkey_to_text(&buf, k.k);
		prt_printf(&buf, " %px\n", k.v);

		panic("%s\n", buf.buf);
	}
#endif
}

static inline struct btree_path_level *insert_l(struct btree_trans *trans, struct btree_insert_entry *i)
{
	return (trans->paths + i->path)->l + i->level;
}

static inline bool same_leaf_as_prev(struct btree_trans *trans,
				     struct btree_insert_entry *i)
{
	return i != trans->updates &&
		insert_l(trans, &i[0])->b == insert_l(trans, &i[-1])->b;
}

static inline bool same_leaf_as_next(struct btree_trans *trans,
				     struct btree_insert_entry *i)
{
	return i + 1 < trans->updates + trans->nr_updates &&
		insert_l(trans, &i[0])->b == insert_l(trans, &i[1])->b;
}

inline void bch2_btree_node_prep_for_write(struct btree_trans *trans,
					   struct btree_path *path,
					   struct btree *b)
{
	struct bch_fs *c = trans->c;

	if (unlikely(btree_node_just_written(b)) &&
	    bch2_btree_post_write_cleanup(c, b))
		bch2_trans_node_reinit_iter(trans, b);

	/*
	 * If the last btree/bset.has been written, or if it's gotten too big - start
	 * a new bset to insert into:
	 */
	if (want_new_bset(c, b))
		bch2_btree_init_next(trans, b);
}

static noinline int trans_lock_write_fail(struct btree_trans *trans, struct btree_insert_entry *i, int ret)
{
	while (--i >= trans->updates) {
		if (same_leaf_as_prev(trans, i))
			continue;

		bch2_btree_node_unlock_write(trans, trans->paths + i->path, insert_l(trans, i)->b);
	}

	return ret;
}

__always_inline
static int bch2_trans_lock_write_inlined(struct btree_trans *trans)
{
	EBUG_ON(trans->write_locked);

	trans_for_each_update(trans, i) {
		if (same_leaf_as_prev(trans, i))
			continue;

		int ret = bch2_btree_node_lock_write(trans, trans->paths + i->path, &insert_l(trans, i)->b->c);
		if (unlikely(ret))
			return trans_lock_write_fail(trans, i, ret);

		if (!i->cached)
			bch2_btree_node_prep_for_write(trans, trans->paths + i->path, insert_l(trans, i)->b);
	}

	trans->write_locked = true;
	return 0;
}

static int bch2_trans_lock_write(struct btree_trans *trans)
{
	return bch2_trans_lock_write_inlined(trans);
}

static inline void bch2_trans_unlock_updates_write(struct btree_trans *trans)
{
	if (likely(trans->write_locked)) {
		trans_for_each_update(trans, i)
			if (btree_node_locked_type(trans->paths + i->path, i->level) ==
			    BTREE_NODE_WRITE_LOCKED)
				bch2_btree_node_unlock_write_inlined(trans,
						trans->paths + i->path, insert_l(trans, i)->b);
		trans->write_locked = false;
	}
}

/* Inserting into a given leaf node (last stage of insert): */

/* Handle overwrites and do insert, for non extents: */
static __always_inline bool
bch2_btree_bset_insert_key_inlined(struct btree_trans *trans,
				   struct btree_path *path,
				   struct btree *b,
				   struct btree_node_iter *node_iter,
				   struct bkey_i *insert)
{
	EBUG_ON(btree_node_just_written(b));
	EBUG_ON(bset_written(b, btree_bset_last(b)));
	EBUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k));
	EBUG_ON(bpos_lt(insert->k.p, b->data->min_key));
	EBUG_ON(bpos_gt(insert->k.p, b->data->max_key));
	EBUG_ON(insert->k.u64s > bch2_btree_keys_u64s_remaining(b));
	EBUG_ON(!b->c.level && !bpos_eq(insert->k.p, path->pos));
	kmsan_check_memory(insert, bkey_bytes(&insert->k));

	struct bkey_packed *k = bch2_btree_node_iter_peek_all(node_iter, b);
	if (k && bkey_cmp_left_packed(b, k, &insert->k.p))
		k = NULL;

	/* @k is the key being overwritten/deleted, if any: */
	EBUG_ON(k && bkey_deleted(k));

	/* Deleting, but not found? nothing to do: */
	if (bkey_deleted(&insert->k) && !k)
		return false;

	bool k_writeable = k >= btree_bset_last(b)->start;

	if (k) {
		btree_account_key_drop(b, k);
		k->type = KEY_TYPE_deleted;

		if (k->needs_whiteout) {
			if (bkey_deleted(&insert->k))
				bch2_push_whiteout(b, insert->k.p);
			else
				insert->k.needs_whiteout = true;
			k->needs_whiteout = false;
		}

		if (!k_writeable)
			bch2_btree_path_fix_key_modified(trans, b, k);
	}

	unsigned clobber_u64s = k_writeable ? k->u64s : 0;

	if (bkey_deleted(&insert->k)) {
		if (k_writeable)
			bch2_bset_delete(b, k, clobber_u64s);
	} else {
		if (!k_writeable)
			k = bch2_btree_node_iter_bset_pos(node_iter, b, bset_tree_last(b));

		bch2_bset_insert(b, k, insert, clobber_u64s);
	}

	unsigned new_u64s = !bkey_deleted(&insert->k) ? k->u64s : 0;
	if (clobber_u64s != new_u64s)
		bch2_btree_node_iter_fix(trans, path, b, node_iter, k, clobber_u64s, new_u64s);

	return true;
}

bool bch2_btree_bset_insert_key(struct btree_trans *trans,
				struct btree_path *path,
				struct btree *b,
				struct btree_node_iter *node_iter,
				struct bkey_i *insert)
{
	return bch2_btree_bset_insert_key_inlined(trans, path, b, node_iter, insert);
}

static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin,
			       unsigned i, u64 seq)
{
	struct bch_fs *c = container_of(j, struct bch_fs, journal);
	struct btree_write *w = container_of(pin, struct btree_write, journal);
	struct btree *b = container_of(w, struct btree, writes[i]);
	unsigned long old, new;
	unsigned idx = w - b->writes;

	/*
	 * The journal pin doesn't pin the node: another thread can write the
	 * node and drop the pin while we run, and reclaim can then reuse the
	 * node's memory. Snapshot the identity to arm the reuse check: gone
	 * already (unhashed nodes can't be dirty), or going while we'd sleep
	 * on the lock (aborting the attempt), means the pinned write is done
	 * and there's nothing to flush.
	 */
	u64 hash_val = READ_ONCE(b->hash_val);
	if (!hash_val)
		return 0;

	CLASS(btree_trans, trans)(c);
	int ret = lockrestart_do(trans, ({
		btree_path_idx_t path_idx;
		int _ret = bch2_btree_node_lock_with_path(trans, &b->c, SIX_LOCK_read,
							  hash_val, &path_idx);
		if (!_ret) {
			old = READ_ONCE(b->flags);
			do {
				new = old;

				if (!(old & (1 << BTREE_NODE_dirty)) ||
				    !!(old & (1 << BTREE_NODE_write_idx)) != idx ||
				    w->journal.seq != seq)
					break;

				new &= ~BTREE_WRITE_TYPE_MASK;
				new |= BTREE_WRITE_journal_reclaim;
				new |= 1 << BTREE_NODE_need_write;
			} while (!try_cmpxchg(&b->flags, &old, new));

			__bch2_btree_node_write(trans, b, BTREE_WRITE_only_if_need);
			bch2_btree_node_unlock_with_path(trans, path_idx, b->c.level);
		}
		_ret;
	}));

	return bch2_err_matches(ret, BCH_ERR_no_btree_node_reused) ? 0 : ret;
}

int bch2_btree_node_flush0(struct journal *j, struct journal_entry_pin *pin, u64 seq)
{
	return __btree_node_flush(j, pin, 0, seq);
}

int bch2_btree_node_flush1(struct journal *j, struct journal_entry_pin *pin, u64 seq)
{
	return __btree_node_flush(j, pin, 1, seq);
}

inline void bch2_btree_add_journal_pin(struct bch_fs *c,
				       struct btree *b, u64 seq)
{
	struct btree_write *w = btree_current_write(b);

	bch2_journal_pin_add(&c->journal, seq, &w->journal,
			     btree_node_write_idx(b) == 0
			     ? bch2_btree_node_flush0
			     : bch2_btree_node_flush1);
}

noinline
static int btree_insert_topology_check(struct btree_trans *trans, struct btree *b)
{
	CLASS(bch_log_msg, msg)(trans->c);
	msg.m.suppress = true;

	int ret = bch2_btree_node_check_topology_msg(trans, b, &msg.m);
	if (ret) {
		prt_str(&msg.m, "Btree update created topology error:\n");
		bch2_trans_updates_to_text(&msg.m, trans);
		bch2_fs_emergency_read_only(trans->c, &msg.m);
	}
	return ret;
}

/**
 * bch2_btree_insert_key_leaf() - insert a key one key into a leaf node
 * @trans:		btree transaction object
 * @path:		path pointing to @insert's pos
 * @insert:		key to insert
 * @journal_seq:	sequence number of journal reservation
 */
inline void bch2_btree_insert_key_leaf(struct btree_trans *trans,
				       struct btree_path *path,
				       struct bkey_i *insert,
				       u64 journal_seq)
{
	struct bch_fs *c = trans->c;
	struct btree *b = path_l(path)->b;
	struct bset_tree *t = bset_tree_last(b);
	struct bset *i = bset(b, t);
	int old_u64s = bset_u64s(t);
	int old_live_u64s = b->nr.live_u64s;

	if (unlikely(!bch2_btree_bset_insert_key_inlined(trans, path, b,
					&path_l(path)->iter, insert)))
		return;

	if (unlikely(b->c.level && btree_insert_topology_check(trans, b)))
		return;

	i->journal_seq = cpu_to_le64(max(journal_seq, le64_to_cpu(i->journal_seq)));

	bch2_btree_add_journal_pin(c, b, journal_seq);

	if (unlikely(!btree_node_dirty(b)))
		bch2_btree_node_set_dirty(c, b);

	int live_u64s_added = (int) b->nr.live_u64s - old_live_u64s;
	int u64s_added = (int) bset_u64s(t) - old_u64s;

	if (b->sib_u64s[0] != U16_MAX && live_u64s_added < 0)
		b->sib_u64s[0] = max(b->nr.live_u64s, (int) b->sib_u64s[0] + live_u64s_added);
	if (b->sib_u64s[1] != U16_MAX && live_u64s_added < 0)
		b->sib_u64s[1] = max(b->nr.live_u64s, (int) b->sib_u64s[1] + live_u64s_added);

	if (u64s_added > live_u64s_added &&
	    unlikely(bch2_maybe_compact_whiteouts(c, b)))
		bch2_trans_node_reinit_iter(trans, b);
}

/* Cached btree updates: */

/* Normal update interface: */

static inline void btree_insert_entry_checks(struct btree_trans *trans,
					     struct btree_insert_entry *i)
{
	struct btree_path *path = trans->paths + i->path;

	BUG_ON(!bpos_eq(i->k->k.p, path->pos));
	BUG_ON(i->cached	!= path->cached);
	BUG_ON(i->cached &&
	       !i->key_cache_already_flushed  &&
	       bkey_deleted(&i->k->k));;
	BUG_ON(i->level		!= path->level);
	BUG_ON(i->btree_id	!= path->btree_id);
	BUG_ON(i->bkey_type	!= __btree_node_type(path->level, path->btree_id));
	EBUG_ON(!i->level &&
		btree_type_has_snapshots(i->btree_id) &&
		!(i->flags & BTREE_UPDATE_internal_snapshot_node) &&
		test_bit(JOURNAL_replay_done, &trans->c->journal.flags) &&
		i->k->k.p.snapshot &&
		bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot) > 0);
}

noinline __cold
static int journal_transaction_names_changed(struct btree_trans *trans)
{
	struct journal *j = &trans->c->journal;

	bch2_journal_res_put(j, &trans->journal_res);
	return btree_trans_restart(trans, BCH_ERR_transaction_restart_journal_overwrites_changed);
}

static __always_inline int bch2_trans_journal_res_get(struct btree_trans *trans,
						      unsigned flags)
{
	struct journal *j = &trans->c->journal;

	try(bch2_journal_res_get(j, &trans->journal_res,
				 trans->journal_u64s, flags, trans));

	if (unlikely(trans->journal_res.has_overwrites !=
		     trans->journal_transaction_names))
		return journal_transaction_names_changed(trans);

	return 0;
}

#define JSET_ENTRY_LOG_U64s		4

static inline int btree_key_can_insert(struct btree_trans *trans,
				       struct btree *b, unsigned u64s)
{
	if (!bch2_btree_node_insert_fits(b, u64s))
		return bch_err_throw(trans->c, btree_insert_btree_node_full);

	return 0;
}

noinline static int
btree_key_can_insert_cached_slowpath(struct btree_trans *trans,
				     enum bch_trans_commit_flags flags,
				     struct btree_path *path, unsigned u64s)
{
	struct bch_fs *c = trans->c;
	struct bkey_cached *ck = (void *) path->l[0].b;
	unsigned watermark = flags & BCH_WATERMARK_MASK;

	if (u64s > ck->u64s) {
		unsigned new_u64s	= roundup_pow_of_two(u64s + u64s / 2);
		struct bkey_i *new_k	= kmalloc(new_u64s * sizeof(u64), GFP_NOWAIT);
		if (unlikely(!new_k)) {
			bch2_trans_unlock_updates_write(trans);
			bch2_trans_unlock(trans);

			new_k = kmalloc(new_u64s * sizeof(u64), GFP_KERNEL);
			if (!new_k) {
				struct bch_fs *c = trans->c;
				bch_err(c, "error allocating memory for key cache key, btree %s u64s %u",
					bch2_btree_id_str(path->btree_id), new_u64s);
				return bch_err_throw(c, ENOMEM_btree_key_cache_insert);
			}

			int ret = bch2_trans_relock(trans) ?:
				  bch2_trans_lock_write(trans);
			if (unlikely(ret)) {
				kfree(new_k);
				return ret;
			}
		}

		memcpy(new_k, ck->k, ck->u64s * sizeof(u64));
		kfree(ck->k);

		trans_for_each_update(trans, i)
			if (i->old_v == &ck->k->v)
				i->old_v = &new_k->v;

		ck->u64s	= new_u64s;
		ck->k		= new_k;
	}

	if (watermark < BCH_WATERMARK_reclaim &&
	    !test_bit(BKEY_CACHED_DIRTY, &ck->flags) &&
	    bch2_btree_key_cache_must_wait(c) &&
	    test_bit(JOURNAL_replay_done, &c->journal.flags))
		return bch_err_throw(c, btree_insert_need_journal_reclaim);

	return 0;
}

static inline int btree_key_can_insert_cached(struct btree_trans *trans,
					      enum bch_trans_commit_flags flags,
					      struct btree_path *path, unsigned u64s)
{
	struct bkey_cached *ck = (void *) path->l[0].b;

	/*
	 * bch2_varint_decode can read past the end of the buffer by at most 7
	 * bytes (it won't be used):
	 */
	u64s += 1;

	return likely(u64s <= ck->u64s && test_bit(BKEY_CACHED_DIRTY, &ck->flags))
		? 0
		: btree_key_can_insert_cached_slowpath(trans, flags, path, u64s);
}

/* Triggers: */

static int run_one_mem_trigger(struct btree_trans *trans,
			       struct btree_insert_entry *i,
			       unsigned flags)
{
	verify_update_old_key(trans, i);

	struct bkey_i deleted;
	bkey_init(&deleted.k);
	deleted.k.p = i->k->k.p;

	struct bkey old_k = i->old_k;
	struct btree_trigger_op op = {
		.btree		= i->btree_id,
		.level		= i->level,
		.old		= (struct bkey_s_c) { &old_k, i->old_v },
		.new		= bkey_i_to_s(i->k),
		.new_buf_u64s	= i->k_buf_u64s,
	};

	const struct bkey_ops *old_ops = bch2_bkey_type_ops(old_k.type);
	const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type);

	/*
	 * Snapshot key accounting is a net old->new delta, so run it once here
	 * with the real old and new keys, before the per-type dispatch below
	 * rewrites op for its insert/overwrite split.
	 */
	if (btree_type_has_snapshots(op.btree)) {
		op.flags = flags;
		try(bch2_trigger_snapshot_nr_keys(trans, op));
	}

	if (old_ops->trigger &&
	    old_ops->trigger == new_ops->trigger) {
		op.flags = flags|BTREE_TRIGGER_insert|BTREE_TRIGGER_overwrite;
		return old_ops->trigger(trans, op);
	}

	if (new_ops->trigger) {
		op.flags = flags|BTREE_TRIGGER_insert;
		op.old = bkey_i_to_s_c(&deleted);
		try(new_ops->trigger(trans, op));
	}

	if (old_ops->trigger) {
		op.flags	= flags|BTREE_TRIGGER_overwrite;
		op.old		= (struct bkey_s_c) { &old_k, i->old_v },
		op.new		= bkey_i_to_s(&deleted);
		op.new_buf_u64s = 0;
		try(old_ops->trigger(trans, op));
	}

	return 0;
}

static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i)
{
	verify_update_old_key(trans, i);

	struct bkey_i deleted;
	bkey_init(&deleted.k);
	deleted.k.p = i->k->k.p;
	/*
	 * Transactional triggers create new btree_insert_entries, so we can't
	 * pass them a pointer to a btree_insert_entry, that memory is going to
	 * move:
	 */
	struct bkey old_k = i->old_k;
	struct btree_trigger_op op = {
		.btree		= i->btree_id,
		.level		= i->level,
		.old		= (struct bkey_s_c) { &old_k, i->old_v },
		.new		= bkey_i_to_s(i->k),
		.new_buf_u64s	= i->k_buf_u64s,
		.flags		= i->flags|BTREE_TRIGGER_transactional,
	};

	const struct bkey_ops *old_ops = bch2_bkey_type_ops(old_k.type);
	const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type);

	if (!i->insert_trigger_run &&
	    !i->overwrite_trigger_run &&
	    old_ops->trigger == new_ops->trigger) {
		i->overwrite_trigger_run = true;
		i->insert_trigger_run = true;
		op.flags |= BTREE_TRIGGER_insert|BTREE_TRIGGER_overwrite;
	} else if (!i->overwrite_trigger_run) {
		i->overwrite_trigger_run = true;
		op.flags |= BTREE_TRIGGER_overwrite;
		op.new = bkey_i_to_s(&deleted);
		op.new_buf_u64s = 0;
	} else if (!i->insert_trigger_run) {
		i->insert_trigger_run = true;
		op.flags |= BTREE_TRIGGER_insert;
		op.old = bkey_i_to_s_c(&deleted);
		old_ops = new_ops;
	}

	try(old_ops->trigger ? old_ops->trigger(trans, op) : 0);

	return btree_type_has_snapshots(op.btree)
		? bch2_trigger_snapshot_nr_keys(trans, op)
		: 0;
}

static int bch2_trans_commit_run_triggers(struct btree_trans *trans)
{
	unsigned sort_id_start = 0;

	while (sort_id_start < trans->nr_updates) {
		unsigned idx, sort_id = trans->updates[sort_id_start].sort_order;
		bool trans_trigger_run;

		/*
		 * For a given btree, this algorithm runs insert triggers before
		 * overwrite triggers: this is so that when extents are being
		 * moved (e.g. by FALLOCATE_FL_INSERT_RANGE), we don't drop
		 * references before they are re-added.
		 *
		 * Running triggers will append more updates to the list of
		 * updates as we're walking it:
		 */
		do {
			trans_trigger_run = false;

			for (idx = sort_id_start;
			     idx < trans->nr_updates && trans->updates[idx].sort_order <= sort_id;
			     idx++) {
				struct btree_insert_entry *i = trans->updates + idx;
				if (i->sort_order < sort_id) {
					sort_id_start = idx;
					continue;
				}

				if ((i->flags & BTREE_TRIGGER_norun) ||
				    !btree_node_type_has_trans_triggers(i->bkey_type) ||
				    (i->insert_trigger_run && i->overwrite_trigger_run))
					continue;

				try(run_one_trans_trigger(trans, i));
				trans_trigger_run = true;
			}
		} while (trans_trigger_run);

		sort_id_start = idx;
	}

#ifdef CONFIG_BCACHEFS_DEBUG
	trans_for_each_update(trans, i)
		BUG_ON(!(i->flags & BTREE_TRIGGER_norun) &&
		       btree_node_type_has_trans_triggers(i->bkey_type) &&
		       (!i->insert_trigger_run || !i->overwrite_trigger_run));
#endif
	return 0;
}

static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans)
{
	trans_for_each_update(trans, i)
		if (btree_node_type_has_triggers(i->bkey_type) &&
		    !(i->flags & BTREE_TRIGGER_norun) &&
		    gc_visited(trans->c, gc_pos_btree(i->btree_id, i->level, i->k->k.p)))
			try(run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_gc));

	return 0;
}

static int bch2_trans_commit_journal_pin_flush(struct journal *j,
				struct journal_entry_pin *_pin, u64 seq)
{
	return 0;
}

noinline __cold
static int trans_commit_accounting_revert(struct btree_trans *trans,
					  enum bch_trans_commit_flags flags,
					  int ret, struct bkey_i *end)
{
	for (struct bkey_i *i = btree_trans_subbuf_base(trans, &trans->accounting);
	     i != end;
	     i = bkey_next(i))
		bch2_accounting_trans_commit_revert(trans, bkey_i_to_accounting(i), flags);
	return ret;
}

noinline __cold
static int trans_commit_fatal_err(struct btree_trans *trans, int ret)
{
	bch2_fs_fatal_error(trans->c, "%s(): fatal error in transaction commit: %s",
			    trans->fn, bch2_err_str(ret));
	return ret;
}

#ifdef CONFIG_BCACHEFS_DEBUG
noinline __cold
static int trans_commit_invalid_entry(struct btree_trans *trans, int ret)
{

	struct bch_fs *c = trans->c;
	CLASS(bch_log_msg, msg)(c);
	prt_printf(&msg.m, "%s(): invalid journal entry on insert\n", trans->fn);
	bch2_trans_updates_to_text(&msg.m, trans);
	bch2_sb_error_count(c, BCH_FSCK_ERR_validate_error_in_commit);
	bch2_fs_emergency_read_only(c, &msg.m);
	bch2_prt_task_backtrace(&msg.m, current, 1, GFP_KERNEL);
	return ret;
}

noinline __cold
static int trans_commit_invalid_key(struct btree_trans *trans,
				    struct btree_insert_entry *i,
				    int ret)
{
	struct bch_fs *c = trans->c;
	CLASS(bch_log_msg, msg)(c);
	prt_printf(&msg.m, "%s() -> %ps: invalid bkey on insert from\n", trans->fn, (void *) i->ip_allocated);
	bch2_trans_updates_to_text(&msg.m, trans);
	bch2_sb_error_count(c, BCH_FSCK_ERR_validate_error_in_commit);
	bch2_fs_emergency_read_only(c, &msg.m);
	bch2_prt_task_backtrace(&msg.m, current, 1, GFP_KERNEL);
	return ret;
}
#endif

__cold
static int bch2_check_drop_overwrites_from_journal(struct btree_trans *trans, bool check)
{
	/*
	 * Accounting keys aren't deduped in the journal: we have to compare
	 * each individual update against what's in the btree to see if it has
	 * been applied yet, and accounting updates also don't overwrite,
	 * they're deltas that accumulate.
	 */
	trans_for_each_update(trans, i)
		if (i->k->k.type != KEY_TYPE_accounting)
			try(bch2_journal_key_check_or_overwrite(trans->c, i->btree_id,
								i->level, i->k->k.p, check));

	for (struct jset_entry *i = btree_trans_journal_entries_start(trans);
	     i != btree_trans_journal_entries_top(trans);
	     i = vstruct_next(i)) {
		if (i->type == BCH_JSET_ENTRY_btree_keys ||
		    i->type == BCH_JSET_ENTRY_write_buffer_keys) {
			jset_entry_for_each_key(i, k)
				try(bch2_journal_key_check_or_overwrite(trans->c, i->btree_id,
									i->level, k->k.p, check));
		}
	}

	return 0;
}

noinline __cold
static int trans_commit_to_journal_replay_pre(struct btree_trans *trans, enum bch_trans_commit_flags flags)
{
	mutex_lock(&trans->c->journal_keys.overwrite_lock);

	if ((flags & BCH_TRANS_COMMIT_journal_replay) &&
	    bch2_check_drop_overwrites_from_journal(trans, true)) {
		mutex_unlock(&trans->c->journal_keys.overwrite_lock);
		bch2_trans_unlock_updates_write(trans);
		return btree_trans_restart(trans, BCH_ERR_transaction_restart_journal_overwrites_changed);
	}

	return 0;
}

noinline __cold
static void trans_commit_to_journal_replay_post(struct btree_trans *trans, int ret)
{
	if (!ret)
		bch2_check_drop_overwrites_from_journal(trans, false);
	mutex_unlock(&trans->c->journal_keys.overwrite_lock);
}

static int journal_reclaim_wait_done(struct bch_fs *c)
{
	try(bch2_journal_error(&c->journal));
	try(bch2_btree_key_cache_wait_done(c));

	journal_reclaim_kick(&c->journal);
	return 0;
}

static bool trans_commit_has_extents(struct btree_trans *trans)
{
	trans_for_each_update(trans, i)
		if (i->btree_id == BTREE_ID_extents ||
		    i->btree_id == BTREE_ID_reflink ||
		    i->btree_id == BTREE_ID_stripes ||
		    i->level)
			return true;
	return false;
}

static int __bch2_trans_commit_error(struct btree_trans *trans, unsigned flags,
				     struct btree_insert_entry *i,
				     int ret, unsigned long trace_ip)
{
	struct bch_fs *c = trans->c;
	enum bch_watermark watermark = flags & BCH_WATERMARK_MASK;

	if (bch2_err_matches(ret, BCH_ERR_journal_res_blocked)) {
		/*
		 * XXX: this should probably be a separate BTREE_INSERT_NONBLOCK
		 * flag
		 */
		if ((flags & BCH_TRANS_COMMIT_journal_reclaim) &&
		    watermark < BCH_WATERMARK_reclaim)
			return bch_err_throw(c, journal_reclaim_would_deadlock);

		return drop_locks_do(trans,
			bch2_trans_journal_res_get(trans,
					(flags & BCH_WATERMARK_MASK)|
					JOURNAL_RES_GET_CHECK));
	}

	switch (ret) {
	case -BCH_ERR_btree_insert_btree_node_full:
		if (trans_commit_has_extents(trans))
			flags = btree_update_set_watermark_hipri(flags);

		ret = bch2_btree_split_leaf(trans, i->path, i->k->k.u64s, flags);
		if (!ret && trans->has_interior_updates)
			return btree_trans_restart(trans,
					     BCH_ERR_transaction_restart_split_with_interior_updates);

		if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
			event_inc_trace(c, trans_restart_btree_node_split, buf, ({
				prt_printf(&buf, "%s\n", trans->fn);
				bch2_btree_path_to_text(&buf, trans, i->path, trans->paths + i->path);
			}));
		return ret;
	case -BCH_ERR_btree_insert_need_mark_replicas:
		return drop_locks_do(trans, bch2_accounting_update_sb(trans));
	case -BCH_ERR_btree_insert_need_journal_reclaim:
		bch2_trans_unlock(trans);

		event_inc_trace(c, trans_blocked_journal_reclaim, buf, ({
			prt_printf(&buf, "%s\n", trans->fn);
			prt_printf(&buf, "key cache dirty %lu/%lu must_wait %zu\n",
				   atomic_long_read(&c->btree.key_cache.nr_keys),
				   atomic_long_read(&c->btree.key_cache.nr_dirty),
				   __bch2_btree_key_cache_must_wait(c));
		}));
		track_event_change(&c->times[BCH_TIME_blocked_key_cache_flush], true);

		if (!wait_event_freezable_timeout(c->journal.reclaim_wait,
						  (ret = journal_reclaim_wait_done(c)),
						  HZ)) {
			bch2_trans_unlock_long(trans);
			wait_event_freezable(c->journal.reclaim_wait,
					     (ret = journal_reclaim_wait_done(c)));
		}

		track_event_change(&c->times[BCH_TIME_blocked_key_cache_flush], false);

		return ret < 0 ? ret : bch2_trans_relock(trans);
	default:
		BUG_ON(ret >= 0);
		return ret;
	}
}

static noinline
int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags,
			    struct btree_insert_entry *i,
			    int ret, unsigned long trace_ip)
{
	ret = __bch2_trans_commit_error(trans, flags, i, ret, trace_ip);

	BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted);

	bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOSPC) &&
				(flags & BCH_TRANS_COMMIT_no_enospc),
				trans->c, "%s: incorrectly got %s\n", __func__, bch2_err_str(ret));
	return ret;
}

/*
 * This is for updates done in the early part of fsck - btree_gc - before we've
 * gone RW. we only add the new key to the list of keys for journal replay to
 * do.
 */
static noinline int
do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans,
				       enum bch_trans_commit_flags flags)
{
	struct bch_fs *c = trans->c;

	/* we're actually shutting down */
	if (test_bit(BCH_FS_may_go_rw, &c->flags))
		return bch_err_throw(c, erofs_trans_commit);

	BUG_ON(current != c->recovery_task);

	struct bkey_i *accounting;
	int ret = 0;
retry:
	memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta));
	/*
	 * Raw lock of the inner rwsem, not the percpu_read_noio guard:
	 * transaction commit runs with the trans locked, so PF_MEMALLOC_NOIO is
	 * already held, and this read section is released mid-function on the
	 * revert paths below, which a guard's scope can't express.
	 */
	percpu_down_read(&c->capacity.mark_lock.lock);
	for (accounting = btree_trans_subbuf_base(trans, &trans->accounting);
	     accounting != btree_trans_subbuf_top(trans, &trans->accounting);
	     accounting = bkey_next(accounting)) {
		ret = likely(!(flags & BCH_TRANS_COMMIT_skip_accounting_apply))
			? bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(accounting),
						  BCH_ACCOUNTING_normal, false)
			: 0;
		if (ret)
			goto revert_fs_usage;
	}

	/* Only fatal errors are possible later, so no need to revert this */
	bch2_trans_account_disk_usage_change(trans);
	percpu_up_read(&c->capacity.mark_lock.lock);

	trans_for_each_update(trans, i) {
		ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->k);
		if (ret)
			goto fatal_err;
	}

	for (struct jset_entry *i = btree_trans_journal_entries_start(trans);
	     i != btree_trans_journal_entries_top(trans);
	     i = vstruct_next(i)) {
		if (i->type == BCH_JSET_ENTRY_btree_keys ||
		    i->type == BCH_JSET_ENTRY_write_buffer_keys) {
			jset_entry_for_each_key(i, k) {
				ret = bch2_journal_key_insert(c, i->btree_id, i->level, k);
				if (ret)
					goto fatal_err;
			}
		}

		if (i->type == BCH_JSET_ENTRY_btree_root) {
			guard(mutex)(&c->btree.cache.root_lock);

			struct btree_root *r = bch2_btree_id_root(c, i->btree_id);

			bkey_copy(&r->key, i->start);
			r->level = i->level;
			r->alive = true;
		}
	}

	for (struct bkey_i *i = btree_trans_subbuf_base(trans, &trans->accounting);
	     i != btree_trans_subbuf_top(trans, &trans->accounting);
	     i = bkey_next(i)) {
		ret = bch2_journal_key_insert(c, BTREE_ID_accounting, 0, i);
		if (ret)
			goto fatal_err;
	}

	event_inc_trace(c, transaction_commit, buf, ({
		prt_printf(&buf, "%s\nseq %llu\n", trans->fn, trans->journal_res.seq);
		bch2_trans_updates_to_text(&buf, trans);
	}));

	return 0;
fatal_err:
	bch2_fs_fatal_error(c, "fatal error in transaction commit: %s", bch2_err_str(ret));
	/* Raw re-take for the revert below; still the trans-locked commit path (NOIO held). */
	percpu_down_read(&c->capacity.mark_lock.lock);
revert_fs_usage:
	for (struct bkey_i *i = btree_trans_subbuf_base(trans, &trans->accounting);
	     i != accounting;
	     i = bkey_next(i))
		bch2_accounting_trans_commit_revert(trans, bkey_i_to_accounting(i), flags);
	percpu_up_read(&c->capacity.mark_lock.lock);

	if (bch2_err_matches(ret, BCH_ERR_btree_insert_need_mark_replicas)) {
		ret = drop_locks_do(trans, bch2_accounting_update_sb(trans));
		if (!ret)
			goto retry;
	}
	return ret;
}

noinline __cold
static int bch2_trans_commit_extra_disk_res(struct btree_trans *trans,
					    enum bch_trans_commit_flags flags)
{
	return bch2_disk_reservation_add(trans->c, trans->disk_res,
				trans->extra_disk_res,
				(flags & BCH_TRANS_COMMIT_no_enospc)
				? BCH_DISK_RESERVATION_NOFAIL : 0);
}

noinline __cold
static void trans_commit_compact_tail(struct btree_trans *trans,
				      struct btree_insert_entry *dst,
				      struct btree_insert_entry *i)
{
	/*
	 * Finish compacting the updates array, copying down the entries we
	 * hadn't scanned yet: called when trans->updates must be made
	 * consistent mid-compaction - on error, so that
	 * bch2_trans_reset_updates() sees each surviving path exactly once
	 * (the noops' paths have already been put, and entries between dst
	 * and i are stale duplicates), and before btree node merge.
	 */
	while (++i < trans->updates + trans->nr_updates)
		*dst++ = *i;
	trans->nr_updates = dst - trans->updates;
}

noinline __cold
static int trans_commit_merge(struct btree_trans *trans,
			      enum bch_trans_commit_flags flags,
			      struct btree_insert_entry **ip,
			      struct btree_insert_entry **dstp)
{
	struct bch_fs *c = trans->c;

	if (!enumerated_ref_tryget(&c->writes, BCH_WRITE_REF_trans))
		return 0;

	/* Put trans->updates in a consistent state in case we error: */
	trans_commit_compact_tail(trans, *dstp, *ip);
	*ip = *dstp - 1;

	unsigned dst_idx = *dstp - trans->updates;

	flags = btree_update_set_watermark_hipri(flags);
	int ret = __bch2_foreground_maybe_merge(trans, (*ip)->path, (*ip)->level, flags, NULL);
	enumerated_ref_put(&c->writes, BCH_WRITE_REF_trans);

	/*
	 * The merge takes paths on the siblings; if that grows the paths
	 * table, trans->updates moves with it - they're a single allocation,
	 * see btree_paths_realloc():
	 */
	*dstp	= trans->updates + dst_idx;
	*ip	= *dstp - 1;

	return ret;
}

noinline __cold
static void transaction_commit_trace(struct btree_trans *trans)
{
	__event_trace(trans->c, transaction_commit, buf, ({
		prt_printf(&buf, "%s\nseq %llu\n", trans->fn, trans->journal_res.seq);
		bch2_trans_updates_to_text(&buf, trans);
	}));
}

static inline bool update_is_noop(struct btree_insert_entry *i, enum bch_trans_commit_flags flags)
{
	struct bkey_s_c old = { &i->old_k, i->old_v };

	/*
	 * We can't drop noop inode updates because fsync relies
	 * on grabbing the journal_seq of the latest update from
	 * the inode - and the journal_seq isn't updated until
	 * the atomic trigger:
	 */
	return likely(!(flags & BCH_TRANS_COMMIT_no_skip_noops) &&
		      i->bkey_type != BKEY_TYPE_inodes &&
		      bkey_and_val_eq(old, bkey_i_to_s_c(i->k)));
}

/*
 * Write buffer btree updates are never authored on their own: they're emitted
 * by a trigger on an update to some other btree. So if update_is_noop() dropped
 * an update and all that's left is write buffer keys, those keys are the
 * dropped update's trigger output - describing a change that isn't happening,
 * and noops themselves.
 *
 * They can't be recognized one at a time the way update_is_noop() does: a write
 * buffer update carries no old key to compare against. @dropped_noops is what
 * makes it decidable - without it this is indistinguishable from a transaction
 * that only ever meant to write buffered keys.
 *
 * Committing them anyway is not just wasted work. bch2_trans_commit_lazy()
 * signals success as transaction_restart_commit, so a caller looping over a
 * write buffer btree re-drives at the same position, cannot see its own
 * buffered write, and reissues the identical commit forever.
 */
static bool commit_became_noop(struct btree_trans *trans, bool dropped_noops)
{
	if (!dropped_noops || trans->nr_updates || trans->accounting.u64s)
		return false;

	for (struct jset_entry *i = btree_trans_journal_entries_start(trans);
	     i != btree_trans_journal_entries_top(trans);
	     i = vstruct_next(i)) {
		if (i->type != BCH_JSET_ENTRY_write_buffer_keys)
			return false;

		/*
		 * fsck repairs accounting directly, not as a consequence of
		 * another key changing: not derived, and not ours to drop.
		 */
		jset_entry_for_each_key(i, k)
			if (k->k.type == KEY_TYPE_accounting)
				return false;
	}

	return true;
}

static inline int
bch2_trans_commit_write_locked(struct btree_trans *trans,
			       enum bch_trans_commit_flags flags,
			       struct btree_insert_entry **stopped_at,
			       unsigned long trace_ip)
{
	struct bch_fs *c = trans->c;
	struct btree_trans_commit_hook *h;
	unsigned u64s = 0;
	int ret = 0;

	bch2_trans_verify_not_unlocked_or_in_restart(trans);
#if 0
	/* todo: bring back dynamic fault injection */
	if (race_fault()) {
		event_inc_trace(c, trans_restart_fault_inject, buf);
		return btree_trans_restart(trans, BCH_ERR_transaction_restart_fault_inject);
	}
#endif
	/*
	 * Check if the insert will fit in the leaf node with the write lock
	 * held, otherwise another thread could write the node changing the
	 * amount of space available:
	 */

	prefetch(&trans->c->journal.flags);

	trans_for_each_update(trans, i) {
		/* Multiple inserts might go to same leaf: */
		if (!same_leaf_as_prev(trans, i))
			u64s = 0;

		u64s += i->k->k.u64s;
		ret = !i->cached
			? btree_key_can_insert(trans, insert_l(trans, i)->b, u64s)
			: btree_key_can_insert_cached(trans, flags, trans->paths + i->path, u64s);
		if (ret) {
			*stopped_at = i;
			return ret;
		}

		i->k->k.needs_whiteout = false;
	}

	/*
	 * Don't get journal reservation until after we know insert will
	 * succeed:
	 */
	if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res)))
		try(bch2_trans_journal_res_get(trans,
				(flags & BCH_WATERMARK_MASK)|
				JOURNAL_RES_GET_NONBLOCK));

	/*
	 * Not allowed to fail after we've gotten our journal reservation - we
	 * have to use it:
	 */

	if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG) &&
	    !(flags & BCH_TRANS_COMMIT_no_journal_res)) {
		if (static_branch_unlikely(&bch2_journal_seq_verify))
			trans_for_each_update(trans, i)
				i->k->k.bversion.lo = trans->journal_res.seq;
		else if (static_branch_unlikely(&bch2_inject_invalid_keys))
			trans_for_each_update(trans, i)
				i->k->k.bversion = MAX_VERSION;
	}

	h = trans->hooks;
	while (h) {
		try(h->fn(trans, h));
		h = h->next;
	}

	scoped_guard(percpu_read_noio, &c->capacity.mark_lock) {
		for (struct bkey_i *accounting = btree_trans_subbuf_base(trans, &trans->accounting);
		     accounting != btree_trans_subbuf_top(trans, &trans->accounting);
		     accounting = bkey_next(accounting)) {
			ret = bch2_accounting_trans_commit_hook(trans,
						bkey_i_to_accounting(accounting), flags);
			if (unlikely(ret))
				return trans_commit_accounting_revert(trans, flags, ret, accounting);
		}

		struct bch_fs_usage_base *d = &trans->fs_usage_delta;
		if (d->hidden | d->btree | d->data | d->cached | d->reserved)
			bch2_trans_account_disk_usage_change(trans);
	}

	trans_for_each_update(trans, i)
		if (btree_node_type_has_atomic_triggers(i->bkey_type) &&
		    !(i->flags & BTREE_TRIGGER_norun)) {
			ret = run_one_mem_trigger(trans, i, BTREE_TRIGGER_atomic|i->flags);
			if (unlikely(ret))
				return trans_commit_fatal_err(trans, ret);
		}

	if (unlikely(c->gc.pos.phase)) {
		ret = bch2_trans_commit_run_gc_triggers(trans);
		if (unlikely(ret))
			return trans_commit_fatal_err(trans, ret);
	}

#ifdef CONFIG_BCACHEFS_DEBUG
	struct bkey_validate_context validate_context = { .from	= BKEY_VALIDATE_commit };

	if (!(flags & BCH_TRANS_COMMIT_no_journal_res))
		validate_context.flags = BCH_VALIDATE_write|BCH_VALIDATE_commit;

	for (struct jset_entry *i = btree_trans_journal_entries_start(trans);
	     i != btree_trans_journal_entries_top(trans);
	     i = vstruct_next(i)) {
		ret = bch2_journal_entry_validate(c, NULL, i,
						  bcachefs_metadata_version_current,
						  CPU_BIG_ENDIAN, validate_context);
		if (unlikely(ret))
			return trans_commit_invalid_entry(trans, ret);
	}

	trans_for_each_update(trans, i) {
		validate_context.level	= i->level;
		validate_context.btree	= i->btree_id;

		ret = bch2_bkey_validate(c, bkey_i_to_s_c(i->k), &validate_context);
		if (unlikely(ret))
			return trans_commit_invalid_key(trans, i, ret);
		btree_insert_entry_checks(trans, i);
	}
#endif

	event_inc_trace_fn(c, transaction_commit, transaction_commit_trace(trans));

	if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) {
		struct journal *j = &c->journal;
		struct jset_entry *start = journal_res_entry(j, &trans->journal_res), *entry = start;

		if (trans->journal_transaction_names) {
			struct jset_entry_log *l =
				container_of(__bch2_journal_add_entry(&entry,
						       BCH_JSET_ENTRY_log, 0, 0,
						       JSET_ENTRY_LOG_U64s),
					     struct jset_entry_log, entry);

			memcpy_and_pad(l->d, JSET_ENTRY_LOG_U64s * sizeof(u64),
				       trans->fn, strlen(trans->fn), 0);
		}

		trans_for_each_update(trans, i) {
			if (i->key_cache_already_flushed)
				continue;

			if (i->flags & BTREE_UPDATE_nojournal)
				continue;

			verify_update_old_key(trans, i);

			if (trans->journal_transaction_names) {
				struct jset_entry *e =
					__bch2_journal_add_entry(&entry,
						       BCH_JSET_ENTRY_overwrite,
						       i->btree_id, i->level,
						       i->old_k.u64s);
				bkey_reassemble((struct bkey_i *) e->start,
						(struct bkey_s_c) { &i->old_k, i->old_v });
			}

			struct jset_entry *e =
				__bch2_journal_add_entry(&entry,
					       BCH_JSET_ENTRY_btree_keys,
					       i->btree_id, i->level,
					       i->k->k.u64s);
			bkey_copy((struct bkey_i *) e->start, i->k);
		}

		memcpy_u64s(entry,
			    btree_trans_journal_entries_start(trans),
			    trans->journal_entries.u64s);

		entry = (struct jset_entry *) ((u64 *) entry + trans->journal_entries.u64s);

		if (trans->accounting.u64s)
			memcpy_u64s(__bch2_journal_add_entry(&entry,
						BCH_JSET_ENTRY_write_buffer_keys,
						BTREE_ID_accounting, 0,
						trans->accounting.u64s)->_data,
				    btree_trans_subbuf_base(trans, &trans->accounting),
				    trans->accounting.u64s);

		/*
		 * Partial revert: advance res->offset/u64s past what the fill
		 * consumed, restoring the old external protocol so
		 * bch2_journal_res_put() (back at its pre-refactor position in
		 * do_bch2_trans_commit) fills only the remaining slack:
		 */
		unsigned consumed = (u64 *) entry - (u64 *) start;
		EBUG_ON(consumed > trans->journal_res.u64s);
		trans->journal_res.offset	+= consumed;
		trans->journal_res.u64s		-= consumed;

		if (trans->journal_seq)
			*trans->journal_seq = trans->journal_res.seq;

		if (unlikely(trans->journal_pin))
			bch2_journal_pin_set(&c->journal, trans->journal_res.seq,
					     trans->journal_pin,
					     bch2_trans_commit_journal_pin_flush);

		if (trans->flush)
			bch2_journal_res_flush(&c->journal, &trans->journal_res, trans->flush);
	} else {
		/* No reservation, so the caller named the seq to pin: */
		trans->journal_res.seq = trans->journal_seq_to_pin;
	}

	trans_for_each_update(trans, i) {
		struct btree_path *path = trans->paths + i->path;

		if (!i->cached)
			bch2_btree_insert_key_leaf(trans, path, i->k, trans->journal_res.seq);
		else if (!i->key_cache_already_flushed)
			bch2_btree_insert_key_cached(trans, i);
		else
			bch2_btree_key_cache_drop(trans, path);
	}

	return 0;
}

/*
 * Get journal reservation, take write locks, and attempt to do btree update(s):
 */
static inline int
do_bch2_trans_commit(struct btree_trans *trans,
		     enum bch_trans_commit_flags flags,
		     struct btree_insert_entry **stopped_at,
		     unsigned long trace_ip)
{
	/*
	 * Pre-grow journal entries subbuf for write buffer updates
	 * that will be added during atomic triggers (can't realloc
	 * after journal reservation):
	 */
	if (trans->extra_journal_u64s)
		try(bch2_trans_subbuf_reserve(trans, &trans->journal_entries,
					      trans->extra_journal_u64s));

	try(bch2_trans_lock_write(trans));

	if (unlikely(trans->journal_replay_not_finished))
		try(trans_commit_to_journal_replay_pre(trans, flags));

	int ret = bch2_trans_commit_write_locked(trans, flags, stopped_at, trace_ip);

	if (trans->journal_replay_not_finished)
		trans_commit_to_journal_replay_post(trans, ret);

	bch2_trans_unlock_updates_write(trans);

	if (unlikely(trans->journal_res.ref))
		bch2_journal_res_put(&trans->c->journal, &trans->journal_res);

	return ret;
}

static noinline int bch2_trans_commit_btree_write_ratelimit(struct btree_trans *trans)
{
	struct bch_fs *c = trans->c;
	struct bch_fs_btree_cache *bc = &c->btree.cache;

	return drop_locks_do(trans, ({
		trans_wait_event(trans, &bc->nr_in_flight_wait,
			atomic_long_read(&bc->nr_in_flight_inner) < BTREE_WRITE_IO_LIMIT(c) * 3 / 4 &&
			!bch2_btree_cache_should_throttle(c));
		0;
	}));
}

int __bch2_trans_commit(struct btree_trans *trans, enum bch_trans_commit_flags flags,
			bool lazy)
{
	struct btree_insert_entry *errored_at = NULL;
	struct bch_fs *c = trans->c;
	int ret = 0;

	bch2_trans_verify_not_unlocked_or_in_restart(trans);

	ret = trans_maybe_inject_restart(trans, _RET_IP_);
	if (unlikely(ret))
		goto out_reset;

	if (!bch2_trans_has_updates(trans))
		goto out_reset;

	if ((flags & BCH_WATERMARK_MASK) <= BCH_WATERMARK_normal &&
	    likely(test_bit(JOURNAL_replay_done, &c->journal.flags)) &&
	    unlikely(bch2_btree_cache_should_throttle(c))) {
		ret = bch2_trans_commit_btree_write_ratelimit(trans);
		if (ret)
			goto out_reset;
	}

	ret = bch2_trans_commit_run_triggers(trans);
	if (ret)
		goto out_reset;

	unsigned journal_u64s = 0;

	trans->journal_transaction_names = READ_ONCE(c->opts.journal_transaction_names);
	if (trans->journal_transaction_names)
		journal_u64s += jset_u64s(JSET_ENTRY_LOG_U64s);

	if (trans->accounting.u64s)
		journal_u64s += jset_u64s(trans->accounting.u64s);

	int u64s_delta = 0;
	bool dropped_noops = false;
	struct btree_insert_entry *dst = trans->updates;
	trans_for_each_update(trans, i) {
		if (unlikely(update_is_noop(i, flags))) {
			bch2_path_put(trans, i->path, true);
			dropped_noops = true;
			continue;
		}

		*dst++ = *i;

		struct btree_path *path = trans->paths + i->path;

		EBUG_ON(!path->should_be_locked);

		ret = bch2_btree_path_upgrade(trans, path, i->level + 1);
		if (unlikely(ret)) {
			trans_commit_compact_tail(trans, dst, i);
			goto out_reset;
		}

		EBUG_ON(!btree_node_intent_locked(path, i->level));

		if (i->key_cache_already_flushed)
			continue;

		if (i->flags & BTREE_UPDATE_nojournal)
			continue;

		/* we're going to journal the key being updated: */
		journal_u64s += jset_u64s(i->k->k.u64s);

		/* and we're also going to log the overwrite: */
		if (trans->journal_transaction_names)
			journal_u64s += jset_u64s(i->old_k.u64s);

		if (!i->cached) {
			u64s_delta += !bkey_deleted(&i->k->k) ? i->k->k.u64s : 0;
			u64s_delta -= i->old_btree_u64s;

			if (!same_leaf_as_next(trans, i) &&
			    !trans->has_interior_updates) {
				struct btree_path *path = trans->paths + i->path;
				struct btree *b = path->l[i->level].b;

				if (unlikely(btree_node_needs_merge(c, b, u64s_delta))) {
					ret = trans_commit_merge(trans, flags, &i, &dst);
					if (ret)
						goto out_reset;
				}

				u64s_delta = 0;
			}
		}
	}

	trans->nr_updates = dst - trans->updates;
	if (!bch2_trans_has_updates(trans) ||
	    commit_became_noop(trans, dropped_noops))
		goto out_reset;

	if (unlikely(trans->extra_disk_res)) {
		ret = bch2_trans_commit_extra_disk_res(trans, flags);
		if (ret)
			goto out_reset;
	}

	if (!(flags & BCH_TRANS_COMMIT_no_check_rw) &&
	    unlikely(!enumerated_ref_tryget(&c->writes, BCH_WRITE_REF_trans))) {
		ret = do_bch2_trans_commit_to_journal_replay(trans, flags);
		goto out_reset;
	}

	EBUG_ON(test_bit(BCH_FS_clean_shutdown, &c->flags));
retry:
	errored_at = NULL;
	bch2_trans_verify_not_unlocked_or_in_restart(trans);
	if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res)))
		memset(&trans->journal_res, 0, sizeof(trans->journal_res));
	memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta));

	trans->journal_u64s = journal_u64s + trans->journal_entries.u64s +
		trans->extra_journal_u64s;

	ret = do_bch2_trans_commit(trans, flags, &errored_at, _RET_IP_);

	/* make sure we didn't drop or screw up locks: */
	bch2_trans_verify_locks(trans);

	if (ret)
		goto err;

	trans->commit_count++;
out:
	if (likely(!(flags & BCH_TRANS_COMMIT_no_check_rw)))
		enumerated_ref_put(&c->writes, BCH_WRITE_REF_trans);

	if (!ret)
		bch2_trans_downgrade(trans);
	/*
	 * Lazy commits signal success via transaction_restart_commit; a real
	 * error must not be masked into a restart, or the caller's restart
	 * loop retries a failing commit forever:
	 */
	if (lazy && !ret)
		ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_commit);
out_reset:
	bch2_trans_reset_updates(trans);

	return ret;
err:
	ret = bch2_trans_commit_error(trans, flags, errored_at, ret, _RET_IP_);
	if (ret)
		goto out;

	goto retry;
}
