bcachefs: New bucket invalidate path

In the old allocator code, preparing an existing empty bucket was part
of the same code path that invalidated buckets containing cached data.
In the new allocator code this is no longer the case: the main allocator
path finds empty buckets (via the new freespace btree), and can't
allocate buckets that contain cached data.

We now need a separate code path to invalidate buckets containing cached
data when we're low on empty buckets, which this patch implements. When
the number of free buckets decreases that triggers the new invalidate
path to run, which uses the LRU btree to pick cached data buckets to
invalidate until we're above our watermark.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

4 files changed, 98 insertions, 0 deletions

diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c
index 9514c2e5f01e..fac9337dc543 100644
--- a/fs/bcachefs/alloc_background.c
+++ b/fs/bcachefs/alloc_background.c
@@ -718,6 +718,86 @@ void bch2_do_discards(struct bch_fs *c)
 		percpu_ref_put(&c->writes);
 }
 
+static int invalidate_one_bucket(struct btree_trans *trans, struct bch_dev *ca)
+{
+	struct bch_fs *c = trans->c;
+	struct btree_iter lru_iter, alloc_iter = { NULL };
+	struct bkey_s_c k;
+	struct bkey_i_alloc_v4 *a;
+	u64 bucket, idx;
+	int ret;
+
+	bch2_trans_iter_init(trans, &lru_iter, BTREE_ID_lru,
+			     POS(ca->dev_idx, 0), 0);
+	k = bch2_btree_iter_peek(&lru_iter);
+	ret = bkey_err(k);
+	if (ret)
+		goto out;
+
+	if (!k.k || k.k->p.inode != ca->dev_idx)
+		goto out;
+
+	if (bch2_fs_inconsistent_on(k.k->type != KEY_TYPE_lru, c,
+				    "non lru key in lru btree"))
+		goto out;
+
+	idx	= k.k->p.offset;
+	bucket	= le64_to_cpu(bkey_s_c_to_lru(k).v->idx);
+
+	a = bch2_trans_start_alloc_update(trans, &alloc_iter,
+					  POS(ca->dev_idx, bucket));
+	ret = PTR_ERR_OR_ZERO(a);
+	if (ret)
+		goto out;
+
+	if (bch2_fs_inconsistent_on(idx != alloc_lru_idx(a->v), c,
+			"invalidating bucket with wrong lru idx (got %llu should be %llu",
+			idx, alloc_lru_idx(a->v)))
+		goto out;
+
+	SET_BCH_ALLOC_V4_NEED_INC_GEN(&a->v, false);
+	a->v.gen++;
+	a->v.data_type		= 0;
+	a->v.dirty_sectors	= 0;
+	a->v.cached_sectors	= 0;
+	a->v.io_time[READ]	= atomic64_read(&c->io_clock[READ].now);
+	a->v.io_time[WRITE]	= atomic64_read(&c->io_clock[WRITE].now);
+
+	ret = bch2_trans_update(trans, &alloc_iter, &a->k_i,
+				BTREE_TRIGGER_BUCKET_INVALIDATE);
+out:
+	bch2_trans_iter_exit(trans, &alloc_iter);
+	bch2_trans_iter_exit(trans, &lru_iter);
+	return ret;
+}
+
+static void bch2_do_invalidates_work(struct work_struct *work)
+{
+	struct bch_fs *c = container_of(work, struct bch_fs, invalidate_work);
+	struct bch_dev *ca;
+	struct btree_trans trans;
+	unsigned i;
+	int ret = 0;
+
+	bch2_trans_init(&trans, c, 0, 0);
+
+	for_each_member_device(ca, c, i)
+		while (!ret && should_invalidate_buckets(ca))
+			ret = __bch2_trans_do(&trans, NULL, NULL,
+					      BTREE_INSERT_USE_RESERVE|
+					      BTREE_INSERT_NOFAIL,
+					invalidate_one_bucket(&trans, ca));
+
+	bch2_trans_exit(&trans);
+	percpu_ref_put(&c->writes);
+}
+
+void bch2_do_invalidates(struct bch_fs *c)
+{
+	if (percpu_ref_tryget(&c->writes))
+		queue_work(system_long_wq, &c->invalidate_work);
+}
+
 static int bch2_dev_freespace_init(struct bch_fs *c, struct bch_dev *ca)
 {
 	struct btree_trans trans;
@@ -1002,4 +1082,5 @@ void bch2_fs_allocator_background_init(struct bch_fs *c)
 {
 	spin_lock_init(&c->freelist_lock);
 	INIT_WORK(&c->discard_work, bch2_do_discards_work);
+	INIT_WORK(&c->invalidate_work, bch2_do_invalidates_work);
 }
diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h
index 8ba9bf853c2f..d4883d3cd642 100644
--- a/fs/bcachefs/alloc_background.h
+++ b/fs/bcachefs/alloc_background.h
@@ -115,6 +115,17 @@ int bch2_trans_mark_alloc(struct btree_trans *, struct bkey_s_c,
 			  struct bkey_i *, unsigned);
 void bch2_do_discards(struct bch_fs *);
 
+static inline bool should_invalidate_buckets(struct bch_dev *ca)
+{
+	struct bch_dev_usage u = bch2_dev_usage_read(ca);
+
+	return u.d[BCH_DATA_cached].buckets &&
+		u.buckets_unavailable + u.d[BCH_DATA_cached].buckets <
+		ca->mi.nbuckets >> 7;
+}
+
+void bch2_do_invalidates(struct bch_fs *);
+
 int bch2_fs_freespace_init(struct bch_fs *);
 
 void bch2_recalc_capacity(struct bch_fs *);
diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index ca48b3f86304..66d9c209252e 100644
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -759,6 +759,7 @@ struct bch_fs {
 
 	struct buckets_waiting_for_journal buckets_waiting_for_journal;
 	struct work_struct	discard_work;
+	struct work_struct	invalidate_work;
 
 	/* GARBAGE COLLECTION */
 	struct task_struct	*gc_thread;
diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index 0e86b45b6c55..bfab5d88550b 100644
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -548,6 +548,11 @@ int bch2_mark_alloc(struct btree_trans *trans,
 	    !new_a.journal_seq)
 		bch2_do_discards(c);
 
+	if (!old_a.data_type &&
+	    new_a.data_type &&
+	    should_invalidate_buckets(ca))
+		bch2_do_invalidates(c);
+
 	if (bucket_state(new_a) == BUCKET_need_gc_gens) {
 		atomic_inc(&c->kick_gc);
 		wake_up_process(c->gc_thread);