/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_SB_MEMBERS_H
#define _BCACHEFS_SB_MEMBERS_H

#include "btree/bkey_types.h"
#include "util/enumerated_ref.h"
#include "util/darray.h"

/* Superblock member access: */

extern char * const bch2_member_error_strs[];

static inline struct bch_member *
__bch2_members_v2_get_mut(struct bch_sb_field_members_v2 *mi, unsigned i)
{
	return (void *) mi->_members + (i * le16_to_cpu(mi->member_bytes));
}

static inline struct bch_member bch2_members_v2_get(struct bch_sb_field_members_v2 *mi, int i)
{
	struct bch_member ret, *p = __bch2_members_v2_get_mut(mi, i);
	memset(&ret, 0, sizeof(ret));
	memcpy(&ret, p, min_t(size_t, le16_to_cpu(mi->member_bytes), sizeof(ret)));
	return ret;
}

static inline struct bch_member *members_v1_get_mut(struct bch_sb_field_members_v1 *mi, int i)
{
	return (void *) mi->_members + (i * BCH_MEMBER_V1_BYTES);
}

static inline struct bch_member bch2_members_v1_get(struct bch_sb_field_members_v1 *mi, int i)
{
	struct bch_member ret, *p = members_v1_get_mut(mi, i);
	memset(&ret, 0, sizeof(ret));
	memcpy(&ret, p, min_t(size_t, BCH_MEMBER_V1_BYTES, sizeof(ret)));
	return ret;
}

int bch2_sb_members_v2_init(struct bch_fs *c);
int bch2_sb_members_cpy_v2_v1(struct bch_sb_handle *disk_sb);
struct bch_member *bch2_members_v2_get_mut(struct bch_sb *sb, int i);
struct bch_member bch2_sb_member_get(struct bch_sb *sb, int i);

void bch2_member_to_text(struct printbuf *, struct bch_member *,
			 struct bch_sb_field_disk_groups *,
			 struct bch_sb *, unsigned);

void bch2_member_to_text_short_locked(struct printbuf *, struct bch_fs *, struct bch_dev *);
void bch2_member_to_text_short(struct printbuf *, struct bch_fs *, struct bch_dev *);
void bch2_devs_mask_to_text_locked(struct printbuf *, struct bch_fs *, struct bch_devs_mask *);

unsigned long bch2_dev_latency_max(struct bch_fs *, struct bch_devs_mask *, int);

/* Device online state: */

static inline bool bch2_dev_is_online(struct bch_dev *ca)
{
	return !enumerated_ref_is_zero(&ca->io_ref[READ]);
}

static inline struct bch_dev *bch2_dev_rcu_noerror(const struct bch_fs *, unsigned);

static inline bool bch2_dev_idx_is_online(struct bch_fs *c, unsigned dev)
{
	return test_bit(dev, c->devs_online.d);
}

/* Device masks and lists: */

static inline unsigned dev_mask_nr(const struct bch_devs_mask *devs)
{
	return bitmap_weight(devs->d, BCH_SB_MEMBERS_MAX);
}

static inline bool bch2_dev_list_has_dev(struct bch_devs_list devs,
					 unsigned dev)
{
	return darray_find(devs, dev) != NULL;
}

static inline void bch2_dev_list_drop_dev(struct bch_devs_list *devs,
					  unsigned dev)
{
	darray_for_each(*devs, i)
		if (*i == dev) {
			darray_remove_item(devs, i);
			return;
		}
}

static inline void bch2_dev_list_add_dev(struct bch_devs_list *devs,
					 unsigned dev)
{
	if (!bch2_dev_list_has_dev(*devs, dev)) {
		BUG_ON(devs->nr >= ARRAY_SIZE(devs->data));
		devs->data[devs->nr++] = dev;
	}
}

static inline struct bch_devs_list bch2_dev_list_single(unsigned dev)
{
	return (struct bch_devs_list) { .nr = 1, .data[0] = dev };
}

/* Device iteration (RCU): */

static inline struct bch_dev *__bch2_next_dev_idx(struct bch_fs *c, unsigned idx,
						  const struct bch_devs_mask *mask)
{
	struct bch_dev *ca = NULL;

	while ((idx = mask
		? find_next_bit(mask->d, c->sb.nr_devices, idx)
		: idx) < c->sb.nr_devices &&
	       !(ca = rcu_dereference_check(c->devs[idx],
					    lockdep_is_held(&c->state_lock))))
		idx++;

	return ca;
}

static inline struct bch_dev *__bch2_next_dev(struct bch_fs *c, struct bch_dev *ca,
					      const struct bch_devs_mask *mask)
{
	return __bch2_next_dev_idx(c, ca ? ca->dev_idx + 1 : 0, mask);
}

#define for_each_member_device_rcu(_c, _ca, _mask)			\
	for (struct bch_dev *_ca = NULL;				\
	     (_ca = __bch2_next_dev((_c), _ca, (_mask)));)

#define for_each_online_member_rcu(_c, _ca)				\
	for_each_member_device_rcu(_c, _ca, &(_c)->devs_online)

#define for_each_rw_member_rcu(_c, _ca)					\
	for_each_member_device_rcu(_c, _ca, &(_c)->allocator.rw_devs[BCH_DATA_free])

/* Device refcounting: */

static inline void bch2_dev_get(struct bch_dev *ca)
{
#ifdef CONFIG_BCACHEFS_DEBUG
	BUG_ON(atomic_long_inc_return(&ca->ref) <= 1L);
#else
	percpu_ref_get(&ca->ref);
#endif
}

static inline void __bch2_dev_put(struct bch_dev *ca)
{
#ifdef CONFIG_BCACHEFS_DEBUG
	long r = atomic_long_dec_return(&ca->ref);
	if (r < (long) !ca->dying)
		panic("bch_dev->ref underflow, last put: %pS\n", (void *) ca->last_put);
	ca->last_put = _THIS_IP_;
	if (!r)
		complete(&ca->ref_completion);
#else
	percpu_ref_put(&ca->ref);
#endif
}

static inline void bch2_dev_put(struct bch_dev *ca)
{
	if (!IS_ERR_OR_NULL(ca))
		__bch2_dev_put(ca);
}
DEFINE_FREE(bch2_dev_put, struct bch_dev *, bch2_dev_put(_T))

/*
 * Memory-lifetime-only reference: keeps the bch_dev allocation alive, but
 * does NOT imply the device is still a member of the filesystem - removal
 * can complete while you hold this. Use it (never ca->ref) in any context
 * that blocks on state_lock, and recheck ca->removing under the lock
 * before touching member state. Drained by bch2_dev_free(), outside
 * state_lock. See the comment at ca->ref_outer.
 */
static inline void bch2_dev_get_outer(struct bch_dev *ca)
{
	refcount_inc(&ca->ref_outer);
}

static inline void bch2_dev_put_outer(struct bch_dev *ca)
{
	if (!IS_ERR_OR_NULL(ca) &&
	    refcount_dec_and_test(&ca->ref_outer))
		complete(&ca->ref_outer_completion);
}

/* Device iteration (refcounted): */

static inline struct bch_dev *bch2_get_next_dev(struct bch_fs *c, struct bch_dev *ca)
{
	guard(rcu)();
	bch2_dev_put(ca);
	if ((ca = __bch2_next_dev(c, ca, NULL)))
		bch2_dev_get(ca);
	return ca;
}

#define for_each_member_device(_c, _ca)					\
	for (struct bch_dev *_ca __free(bch2_dev_put) = NULL;		\
	     (_ca = bch2_get_next_dev(_c, _ca));)

static inline struct bch_dev *bch2_get_next_online_dev(struct bch_fs *c,
						       struct bch_dev *ca,
						       unsigned state_mask,
						       int rw, unsigned ref_idx)
{
	guard(rcu)();
	if (ca)
		enumerated_ref_put(&ca->io_ref[rw], ref_idx);

	while ((ca = __bch2_next_dev(c, ca, NULL)) &&
	       (!((1 << ca->mi.state) & state_mask) ||
		!enumerated_ref_tryget(&ca->io_ref[rw], ref_idx)))
		;

	return ca;
}

#define __for_each_online_member(_c, _ca, state_mask, rw, ref_idx)	\
	for (struct bch_dev *_ca = NULL;				\
	     (_ca = bch2_get_next_online_dev(_c, _ca, state_mask, rw, ref_idx));)

#define for_each_online_member(c, ca, ref_idx)				\
	__for_each_online_member(c, ca, ~0, READ, ref_idx)

#define for_each_rw_member(c, ca, ref_idx)					\
	__for_each_online_member(c, ca, BIT(BCH_MEMBER_STATE_rw), WRITE, ref_idx)

#define for_each_readable_member(c, ca, ref_idx)				\
	__for_each_online_member(c, ca,	BIT( BCH_MEMBER_STATE_rw)|BIT(BCH_MEMBER_STATE_ro), READ, ref_idx)

/* Device lookup: */

static inline bool bch2_dev_exists(const struct bch_fs *c, unsigned dev)
{
	return dev < c->sb.nr_devices && c->devs[dev];
}

static inline bool bucket_valid(const struct bch_dev *ca, u64 b)
{
	return b - ca->mi.first_bucket < ca->mi.nbuckets_minus_first;
}

static inline struct bch_dev *bch2_dev_have_ref(const struct bch_fs *c, unsigned dev)
{
	if (WARN(!bch2_dev_exists(c, dev), "nonexistent dev %u (nr_devices %u)", dev, c->sb.nr_devices))
		return NULL;

	return rcu_dereference_check(c->devs[dev], 1);
}

static inline struct bch_dev *bch2_dev_locked(struct bch_fs *c, unsigned dev)
{
	EBUG_ON(!bch2_dev_exists(c, dev));

	return rcu_dereference_protected(c->devs[dev],
					 lockdep_is_held(&c->sb_lock.lock) ||
					 lockdep_is_held(&c->state_lock));
}

static inline struct bch_dev *bch2_dev_rcu_noerror(const struct bch_fs *c, unsigned dev)
{
	return c && dev < c->sb.nr_devices
		? rcu_dereference(c->devs[dev])
		: NULL;
}

static inline bool bch2_dev_bad_or_evacuating_rcu(struct bch_fs *c, unsigned dev)
{
	struct bch_dev *ca = bch2_dev_rcu_noerror(c, dev);
	return !ca || ca->mi.state == BCH_MEMBER_STATE_evacuating;
}

static inline bool bch2_dev_bad_or_evacuating(struct bch_fs *c, unsigned dev)
{
	guard(rcu)();
	return bch2_dev_bad_or_evacuating_rcu(c, dev);
}

int bch2_dev_missing_bkey_msg(struct bch_fs *, struct bkey_s_c, unsigned, struct printbuf *out);
int bch2_dev_missing_bkey(struct bch_fs *, struct bkey_s_c, unsigned);

void bch2_dev_missing_atomic(struct bch_fs *, unsigned);

static inline struct bch_dev *bch2_dev_rcu(struct bch_fs *c, unsigned dev)
{
	struct bch_dev *ca = bch2_dev_rcu_noerror(c, dev);
	if (unlikely(!ca))
		bch2_dev_missing_atomic(c, dev);
	return ca;
}

static inline struct bch_dev *bch2_dev_tryget_noerror(struct bch_fs *c, unsigned dev)
{
	guard(rcu)();
	struct bch_dev *ca = bch2_dev_rcu_noerror(c, dev);
	if (ca)
		bch2_dev_get(ca);
	return ca;
}

DEFINE_CLASS(bch2_dev_tryget_noerror, struct bch_dev *,
	     bch2_dev_put(_T), bch2_dev_tryget_noerror(c, dev),
	     struct bch_fs *c, unsigned dev);

static inline struct bch_dev *bch2_dev_tryget(struct bch_fs *c, unsigned dev)
{
	struct bch_dev *ca = bch2_dev_tryget_noerror(c, dev);
	if (unlikely(!ca))
		bch2_dev_missing_atomic(c, dev);
	return ca;
}

static inline struct bch_dev *bch2_dev_bkey_tryget(struct bch_fs *c, struct bkey_s_c k, unsigned dev)
{
	struct bch_dev *ca = bch2_dev_tryget_noerror(c, dev);
	if (unlikely(!ca && dev != BCH_SB_MEMBER_INVALID))
		bch2_dev_missing_bkey(c, k, dev);
	return ca;
}

DEFINE_CLASS(bch2_dev_bkey_tryget, struct bch_dev *,
	     bch2_dev_put(_T), bch2_dev_bkey_tryget(c, k, dev),
	     struct bch_fs *c, struct bkey_s_c k, unsigned dev);

static inline struct bch_dev *bch2_dev_bucket_tryget_noerror(struct bch_fs *c, struct bpos bucket)
{
	struct bch_dev *ca = bch2_dev_tryget_noerror(c, bucket.inode);
	if (ca && unlikely(!bucket_valid(ca, bucket.offset))) {
		bch2_dev_put(ca);
		ca = NULL;
	}
	return ca;
}

DEFINE_CLASS(bch2_dev_bucket_tryget_noerror, struct bch_dev *,
	     bch2_dev_put(_T), bch2_dev_bucket_tryget_noerror(c, bucket),
	     struct bch_fs *c, struct bpos bucket);

void bch2_dev_bucket_missing(struct bch_dev *, u64);

static inline struct bch_dev *bch2_dev_bucket_tryget(struct bch_fs *c, struct bpos bucket)
{
	struct bch_dev *ca = bch2_dev_tryget(c, bucket.inode);
	if (ca && unlikely(!bucket_valid(ca, bucket.offset))) {
		bch2_dev_bucket_missing(ca, bucket.offset);
		bch2_dev_put(ca);
		ca = NULL;
	}
	return ca;
}

DEFINE_CLASS(bch2_dev_bucket_tryget, struct bch_dev *,
	     bch2_dev_put(_T), bch2_dev_bucket_tryget(c, bucket),
	     struct bch_fs *c, struct bpos bucket);

static inline struct bch_dev *bch2_dev_iterate_noerror(struct bch_fs *c, struct bch_dev *ca, unsigned dev_idx)
{
	if (ca && ca->dev_idx == dev_idx)
		return ca;
	bch2_dev_put(ca);
	return bch2_dev_tryget_noerror(c, dev_idx);
}

static inline struct bch_dev *bch2_dev_iterate(struct bch_fs *c, struct bch_dev *ca, unsigned dev_idx)
{
	if (ca && ca->dev_idx == dev_idx)
		return ca;
	bch2_dev_put(ca);
	return bch2_dev_tryget(c, dev_idx);
}

static inline struct bch_dev *bch2_dev_get_ioref(struct bch_fs *c, unsigned dev,
						 int rw, unsigned ref_idx)
{
	might_sleep();

	guard(rcu)();
	struct bch_dev *ca = bch2_dev_rcu_noerror(c, dev);
	if (!ca || !enumerated_ref_tryget(&ca->io_ref[rw], ref_idx))
		return NULL;

	if (ca->mi.state == BCH_MEMBER_STATE_rw || rw == READ)
		return ca;

	enumerated_ref_put(&ca->io_ref[rw], ref_idx);
	return NULL;
}

/* Member properties: */

extern const struct bch_sb_field_ops bch_sb_field_ops_members_v1;
extern const struct bch_sb_field_ops bch_sb_field_ops_members_v2;

static inline bool bch2_member_alive(struct bch_member *m)
{
	return  !bch2_is_zero(&m->uuid, sizeof(m->uuid)) &&
		!uuid_equal(&m->uuid, &BCH_SB_MEMBER_DELETED_UUID);
}

static inline bool bch2_member_exists(struct bch_sb *sb, unsigned dev)
{
	if (dev < sb->nr_devices) {
		struct bch_member m = bch2_sb_member_get(sb, dev);
		return bch2_member_alive(&m);
	}
	return false;
}

unsigned bch2_sb_nr_devices(const struct bch_sb *);

static inline struct bch_member_cpu bch2_mi_to_cpu(struct bch_member *mi)
{
	return (struct bch_member_cpu) {
		.nbuckets	= le64_to_cpu(mi->nbuckets),
		.nbuckets_minus_first = le64_to_cpu(mi->nbuckets) -
			le16_to_cpu(mi->first_bucket),
		.first_bucket	= le16_to_cpu(mi->first_bucket),
		.bucket_size	= le16_to_cpu(mi->bucket_size),
		.group		= BCH_MEMBER_GROUP(mi),
		/* .failure_domain is interned in bch2_sb_members_to_cpu() */
		.state		= BCH_MEMBER_STATE(mi),
		.discard	= BCH_MEMBER_DISCARD(mi),
		.data_allowed	= BCH_MEMBER_DATA_ALLOWED(mi),
		.durability	= BCH_MEMBER_DURABILITY(mi)
			? BCH_MEMBER_DURABILITY(mi) - 1
			: 1,
		.freespace_initialized = BCH_MEMBER_FREESPACE_INITIALIZED(mi),
		.initialized		= BCH_MEMBER_INITIALIZED(mi),
		.resize_on_mount	= BCH_MEMBER_RESIZE_ON_MOUNT(mi),
		.rotational		= BCH_MEMBER_ROTATIONAL(mi),
		.valid			= bch2_member_alive(mi),
		.btree_bitmap_shift	= mi->btree_bitmap_shift,
		.btree_allocated_bitmap = le64_to_cpu(mi->btree_allocated_bitmap),
	};
}

void bch2_sb_members_from_cpu(struct bch_fs *);
void bch2_sb_members_to_cpu(struct bch_fs *);

void bch2_dev_io_errors_to_text(struct printbuf *, struct bch_dev *);
void bch2_dev_errors_reset(struct bch_dev *);

/* Btree allocation bitmap: */

static inline bool __bch2_dev_btree_bitmap_marked_sectors(struct bch_dev *ca, u64 start,
							  unsigned sectors, bool with_gc)
{
	u64 end = start + sectors;

	if (end > 64ULL << ca->mi.btree_bitmap_shift)
		return false;

	for (unsigned bit = start >> ca->mi.btree_bitmap_shift;
	     (u64) bit << ca->mi.btree_bitmap_shift < end;
	     bit++)
		if (!(BIT_ULL(bit) &
		      ca->mi.btree_allocated_bitmap &
		      (with_gc
		       ? ca->btree_allocated_bitmap_gc
		       : ~0ULL)))
			return false;
	return true;
}

static inline bool bch2_dev_btree_bitmap_marked_sectors(struct bch_dev *ca, u64 start,
							unsigned sectors)
{
	return __bch2_dev_btree_bitmap_marked_sectors(ca, start, sectors, false);
}

static inline bool bch2_dev_btree_bitmap_marked_sectors_any(struct bch_dev *ca, u64 start, unsigned sectors)
{
	u64 end = start + sectors;

	if (start >= 64ULL << ca->mi.btree_bitmap_shift)
		return false;

	for (unsigned bit = start >> ca->mi.btree_bitmap_shift;
	     (u64) bit << ca->mi.btree_bitmap_shift < end;
	     bit++)
		if (ca->mi.btree_allocated_bitmap & BIT_ULL(bit))
			return true;
	return false;
}

bool bch2_dev_btree_bitmap_marked(struct bch_fs *, struct bkey_s_c);
bool bch2_dev_btree_bitmap_marked_nogc(struct bch_fs *, struct bkey_s_c);

void bch2_dev_btree_bitmap_mark_locked(struct bch_fs *, struct bkey_s_c, bool *);
void bch2_dev_btree_bitmap_mark(struct bch_fs *, struct bkey_s_c);

int bch2_btree_bitmap_gc(struct bch_fs *);
void bch2_maybe_schedule_btree_bitmap_gc_stop(struct bch_fs *);
void bch2_maybe_schedule_btree_bitmap_gc(struct bch_fs *);

/* Member management: */

int bch2_sb_member_alloc(struct bch_fs *);
void bch2_sb_members_clean_deleted(struct bch_fs *);

struct bch_dev_identity {
	char name[sizeof(((struct bch_member *) NULL)->device_name) + 1];
	char model[sizeof(((struct bch_member *) NULL)->device_model) + 1];
	char serial[sizeof(((struct bch_member *) NULL)->device_serial) + 1];
	bool rotational;
};

void bch2_dev_mi_field_read(struct bch_dev *, struct bch_dev_identity *);
void bch2_dev_mi_field_upgrades_locked(struct bch_fs *, struct bch_dev *,
				       const struct bch_dev_identity *, bool *);
void bch2_dev_mi_field_upgrades(struct bch_dev *);
void bch2_fs_mi_field_upgrades(struct bch_fs *);

static inline void bch2_prt_member_name(struct printbuf *out, struct bch_fs *c, unsigned idx)
{
	if (idx == BCH_SB_MEMBER_INVALID) {
		prt_str(out, "(none)");
	} else {
		guard(rcu)();
		guard(printbuf_atomic)(out);
		struct bch_dev *ca = c ? bch2_dev_rcu_noerror(c, idx) : NULL;
		if (ca)
			prt_str(out, ca->name);
		else
			prt_printf(out, "(invalid device %u)", idx);
	}
}

#endif /* _BCACHEFS_SB_MEMBERS_H */
