wait_var_event(&bg->reservations, !atomic_read(&bg->reservations));
}
+/*
+ * Open stripe runs
+ * ================
+ *
+ * The raid56 stripe allocation policy only allocates from "open stripe
+ * runs": contiguous, stripe-aligned regions claimed whole from the free
+ * space cache (btrfs_claim_free_stripe_run()) and filled strictly
+ * sequentially. A run stops accepting allocations when it is exhausted,
+ * when an allocation does not fit in its remainder, or when it is retired
+ * at transaction commit; a closed run is never reopened, and its
+ * unallocated tail returns to the free space cache where, being part of a
+ * partially filled stripe, it cannot be claimed again until the whole
+ * stripe frees. This guarantees a full stripe only receives writes
+ * between two commits of the same transaction window, which is what
+ * closes the raid56 write hole for these block groups.
+ *
+ * Each run counts the bytes reserved from it whose data IO has not yet
+ * completed (inflight_bytes). At commit time, once the transaction can
+ * no longer accept joins (TRANS_STATE_COMMIT_DOING with a single writer
+ * left), btrfs_retire_open_stripes() closes every run opened before the
+ * commit and waits for their inflight bytes to drain. At that point
+ * every extent the committing transaction references sits in a stripe
+ * that will never be written again, and every write to such a stripe has
+ * reached disk before the superblock is written. This is a pure data-IO
+ * wait: it must never wait for ordered extent *completion*, whose
+ * processing needs a transaction join that is blocked at this stage.
+ *
+ * Allocations racing with the commit open new runs stamped with a newer
+ * open_seq and are not retired or waited for: metadata referencing them
+ * can only join the next transaction, so they belong to the next window.
+ * The retire walk may only rely on this for allocations whose block group
+ * membership (fs_info->open_stripe_bgs) was established before the walk
+ * collected its worklist; btrfs_alloc_from_open_stripe() therefore adds
+ * the membership before returning, i.e. before the caller can create the
+ * ordered extent for the allocation.
+ *
+ * Lock order: fs_info->open_stripe_lock outside bg->stripe_run_lock. The
+ * stripe_run_lock is irq-safe because completed data IO is reported from
+ * bio end_io context. Membership in fs_info->open_stripe_bgs holds a
+ * block group reference and is removed only by the retire walk once the
+ * group has no runs left.
+ */
+
+struct btrfs_open_stripe_run {
+ struct list_head list; /* bg->open_stripe_runs */
+ struct btrfs_block_group *bg;
+ u64 start;
+ u64 end; /* exclusive; shrunk to offset on close */
+ u64 offset; /* next unallocated byte */
+ u64 inflight_bytes; /* reserved bytes with data IO pending */
+ u64 open_seq; /* fs_info->stripe_retire_seq at open */
+ bool open; /* accepting allocations */
+};
+
+static bool open_stripe_run_drained(const struct btrfs_open_stripe_run *run)
+{
+ return !run->open && run->inflight_bytes == 0;
+}
+
+/* Free a drained run. Caller holds bg->stripe_run_lock. */
+static void free_open_stripe_run(struct btrfs_block_group *bg,
+ struct btrfs_open_stripe_run *run)
+{
+ ASSERT(open_stripe_run_drained(run));
+ ASSERT(bg->open_stripe != run);
+ list_del(&run->list);
+ kfree(run);
+ wake_up_var(&bg->open_stripe_runs);
+}
+
+static void open_stripe_add_bg(struct btrfs_fs_info *fs_info,
+ struct btrfs_block_group *bg)
+{
+ spin_lock(&fs_info->open_stripe_lock);
+ if (list_empty(&bg->open_stripe_bg_list)) {
+ btrfs_get_block_group(bg);
+ list_add_tail(&bg->open_stripe_bg_list,
+ &fs_info->open_stripe_bgs);
+ }
+ spin_unlock(&fs_info->open_stripe_lock);
+}
+
+/*
+ * Allocate num_bytes from the block group's open stripe run, opening a new
+ * run via btrfs_claim_free_stripe_run() when there is none or the current
+ * one does not fit. The allocation counts toward the run's inflight bytes
+ * until btrfs_open_stripe_write_done() reports its data IO complete.
+ *
+ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in
+ * this block group can satisfy the allocation, or -ENOMEM.
+ */
+int btrfs_alloc_from_open_stripe(struct btrfs_block_group *bg, u64 num_bytes,
+ u64 *ret_offset)
+{
+ struct btrfs_fs_info *fs_info = bg->fs_info;
+ const u64 fsl = bg->full_stripe_len;
+ struct btrfs_open_stripe_run *new_run = NULL;
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ u64 tail_start = 0;
+ u64 tail_len = 0;
+ u64 start;
+ u64 len;
+ int ret;
+
+ ASSERT(num_bytes);
+again:
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ run = bg->open_stripe;
+ if (run) {
+ if (num_bytes <= run->end - run->offset) {
+ *ret_offset = run->offset;
+ run->offset += num_bytes;
+ run->inflight_bytes += num_bytes;
+ if (run->offset == run->end) {
+ run->open = false;
+ bg->open_stripe = NULL;
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ ret = 0;
+ goto out;
+ }
+ /*
+ * Doesn't fit. Close the run; the unallocated tail becomes
+ * free space in a partially filled stripe and stays
+ * unclaimable until the whole stripe frees.
+ */
+ run->open = false;
+ bg->open_stripe = NULL;
+ tail_start = run->offset;
+ tail_len = run->end - run->offset;
+ run->end = run->offset;
+ if (open_stripe_run_drained(run))
+ free_open_stripe_run(bg, run);
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ if (tail_len) {
+ btrfs_add_free_space(bg, tail_start, tail_len);
+ tail_len = 0;
+ }
+
+ if (!new_run) {
+ new_run = kmalloc(sizeof(*new_run), GFP_NOFS);
+ if (!new_run) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ }
+
+ ret = btrfs_claim_free_stripe_run(bg,
+ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
+ &start, &len);
+ if (ret)
+ goto out;
+ if (len < num_bytes) {
+ /* Only a shorter run is free; put it back untouched. */
+ btrfs_add_free_space(bg, start, len);
+ ret = -ENOSPC;
+ goto out;
+ }
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ if (bg->open_stripe) {
+ /*
+ * Lost a race with another opener. Our claim is still made
+ * of fully-free stripes, so it can simply go back.
+ */
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ btrfs_add_free_space(bg, start, len);
+ goto again;
+ }
+ new_run->bg = bg;
+ new_run->start = start;
+ new_run->end = start + len;
+ new_run->offset = start + num_bytes;
+ new_run->inflight_bytes = num_bytes;
+ new_run->open_seq = READ_ONCE(fs_info->stripe_retire_seq);
+ new_run->open = (new_run->offset != new_run->end);
+ list_add_tail(&new_run->list, &bg->open_stripe_runs);
+ bg->open_stripe = new_run->open ? new_run : NULL;
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+
+ *ret_offset = start;
+ new_run = NULL;
+ ret = 0;
+ /* Must precede the caller's ordered extent creation; see above. */
+ open_stripe_add_bg(fs_info, bg);
+out:
+ kfree(new_run);
+ return ret;
+}
+
+/*
+ * Report completed (or abandoned) data IO for an allocation made with
+ * btrfs_alloc_from_open_stripe(). Every allocated byte must be reported
+ * exactly once, in any number of pieces (e.g. after ordered extent splits).
+ */
+void btrfs_open_stripe_write_done(struct btrfs_block_group *bg, u64 start,
+ u64 num_bytes)
+{
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (start >= run->start && start < run->offset) {
+ ASSERT(start + num_bytes <= run->offset);
+ ASSERT(run->inflight_bytes >= num_bytes);
+ run->inflight_bytes -= num_bytes;
+ if (open_stripe_run_drained(run))
+ free_open_stripe_run(bg, run);
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+}
+
+/*
+ * Pointer-based variant of btrfs_open_stripe_write_done() for callers that
+ * recorded the run at allocation or ordered extent creation time; safe from
+ * bio end_io context. The run pointer stays valid until the reporter's own
+ * bytes are reported: a run with inflight bytes is never freed.
+ */
+void btrfs_open_stripe_write_done_run(struct btrfs_open_stripe_run *run,
+ u64 num_bytes)
+{
+ struct btrfs_block_group *bg = run->bg;
+ unsigned long flags;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ ASSERT(run->inflight_bytes >= num_bytes);
+ run->inflight_bytes -= num_bytes;
+ if (open_stripe_run_drained(run))
+ free_open_stripe_run(bg, run);
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+}
+
+/*
+ * Close the block group's open run if it predates @seq and return its
+ * unallocated tail to the free space cache. Does not wait for inflight
+ * IO. Only bg->open_stripe can ever be open, so a single pass suffices.
+ */
+static void close_block_group_stripe_runs(struct btrfs_block_group *bg,
+ u64 seq)
+{
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ u64 tail_start = 0;
+ u64 tail_len = 0;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ run = bg->open_stripe;
+ if (run && run->open_seq < seq) {
+ ASSERT(run->open);
+ run->open = false;
+ bg->open_stripe = NULL;
+ tail_start = run->offset;
+ tail_len = run->end - run->offset;
+ run->end = run->offset;
+ if (open_stripe_run_drained(run))
+ free_open_stripe_run(bg, run);
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ if (tail_len)
+ btrfs_add_free_space(bg, tail_start, tail_len);
+}
+
+/* No run opened before @seq remains (open or draining). */
+static bool bg_open_stripes_settled(struct btrfs_block_group *bg, u64 seq)
+{
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ bool ret = true;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (run->open_seq < seq) {
+ ret = false;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ return ret;
+}
+
+/*
+ * Close and drain all of a block group's stripe runs. The caller must
+ * prevent new allocations first (e.g. the block group is read-only),
+ * otherwise this can wait forever. Membership in fs_info->open_stripe_bgs
+ * is left for the next retire walk to collect.
+ */
+void btrfs_retire_block_group_stripes(struct btrfs_block_group *bg)
+{
+ close_block_group_stripe_runs(bg, U64_MAX);
+ wait_var_event(&bg->open_stripe_runs,
+ bg_open_stripes_settled(bg, U64_MAX));
+}
+
+/*
+ * Commit-time retirement: close every stripe run opened before this call
+ * and wait for all data IO into the stripes of those runs to complete.
+ * Must run where the committing transaction can no longer accept joins
+ * (TRANS_STATE_COMMIT_DOING, single writer); see the comment at the top.
+ * Calls are serialized by the transaction commit.
+ */
+void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info)
+{
+ struct btrfs_block_group *bg;
+ LIST_HEAD(retire_list);
+ unsigned long flags;
+ u64 seq;
+
+ spin_lock(&fs_info->open_stripe_lock);
+ seq = ++fs_info->stripe_retire_seq;
+ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
+ list_add_tail(&bg->open_stripe_retire_list, &retire_list);
+ spin_unlock(&fs_info->open_stripe_lock);
+
+ list_for_each_entry(bg, &retire_list, open_stripe_retire_list)
+ close_block_group_stripe_runs(bg, seq);
+
+ while (!list_empty(&retire_list)) {
+ bg = list_first_entry(&retire_list, struct btrfs_block_group,
+ open_stripe_retire_list);
+ wait_var_event(&bg->open_stripe_runs,
+ bg_open_stripes_settled(bg, seq));
+ list_del_init(&bg->open_stripe_retire_list);
+
+ /* Drop the membership if the group has no runs left. */
+ spin_lock(&fs_info->open_stripe_lock);
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ if (list_empty(&bg->open_stripe_runs) &&
+ !list_empty(&bg->open_stripe_bg_list)) {
+ list_del_init(&bg->open_stripe_bg_list);
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ spin_unlock(&fs_info->open_stripe_lock);
+ btrfs_put_block_group(bg);
+ } else {
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ spin_unlock(&fs_info->open_stripe_lock);
+ }
+ }
+}
+
struct btrfs_caching_control *btrfs_get_caching_control(
struct btrfs_block_group *cache)
{
refcount_set(&cache->refs, 1);
spin_lock_init(&cache->lock);
+ spin_lock_init(&cache->stripe_run_lock);
init_rwsem(&cache->data_rwsem);
INIT_LIST_HEAD(&cache->list);
INIT_LIST_HEAD(&cache->cluster_list);
+ INIT_LIST_HEAD(&cache->open_stripe_runs);
+ INIT_LIST_HEAD(&cache->open_stripe_bg_list);
+ INIT_LIST_HEAD(&cache->open_stripe_retire_list);
INIT_LIST_HEAD(&cache->bg_list);
INIT_LIST_HEAD(&cache->ro_list);
INIT_LIST_HEAD(&cache->discard_list);
return ret;
}
+/*
+ * Test the open stripe run allocator on top of btrfs_claim_free_stripe_run():
+ * sequential filling of a run, close-on-misfit with the tail returned as
+ * unclaimable partial-stripe free space, inflight accounting via
+ * btrfs_open_stripe_write_done(), and commit-style retirement.
+ */
+static int test_open_stripe(struct btrfs_block_group *cache, u32 sectorsize)
+{
+ struct btrfs_fs_info *fs_info = cache->fs_info;
+ const u64 orig_fsl = cache->full_stripe_len;
+ const u64 fsl = 16 * sectorsize;
+ u64 offset;
+ int ret;
+
+ test_msg("running open stripe run tests");
+ cache->full_stripe_len = fsl;
+
+ /* Empty cache: no stripe run to open. */
+ ret = btrfs_alloc_from_open_stripe(cache, sectorsize, &offset);
+ if (ret != -ENOSPC) {
+ test_err("alloc from empty cache returned %d", ret);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Free space [sectorsize, 4*fsl): the first aligned stripe is #1. */
+ ret = btrfs_add_free_space(cache, sectorsize, 4 * fsl - sectorsize);
+ if (ret) {
+ test_err("error adding free space %d", ret);
+ goto out;
+ }
+
+ /* Sequential allocations share one run. */
+ ret = btrfs_alloc_from_open_stripe(cache, 2 * sectorsize, &offset);
+ if (ret || offset != fsl) {
+ test_err("first alloc wrong: ret %d offset %llu", ret, offset);
+ ret = -EINVAL;
+ goto out;
+ }
+ ret = btrfs_alloc_from_open_stripe(cache, 2 * sectorsize, &offset);
+ if (ret || offset != fsl + 2 * sectorsize) {
+ test_err("second alloc wrong: ret %d offset %llu", ret, offset);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Doesn't fit the remainder: closes the run, opens the next stripe. */
+ ret = btrfs_alloc_from_open_stripe(cache, fsl, &offset);
+ if (ret || offset != 2 * fsl) {
+ test_err("misfit alloc wrong: ret %d offset %llu", ret, offset);
+ ret = -EINVAL;
+ goto out;
+ }
+ /* The closed run's tail went back to the free space cache... */
+ if (!test_check_exists(cache, fsl + 4 * sectorsize,
+ fsl - 4 * sectorsize)) {
+ test_err("closed run tail missing from free space");
+ ret = -EINVAL;
+ goto out;
+ }
+ /* ...but is part of a partial stripe, so the next alloc skips it. */
+ ret = btrfs_alloc_from_open_stripe(cache, sectorsize, &offset);
+ if (ret || offset != 3 * fsl) {
+ test_err("post-close alloc wrong: ret %d offset %llu",
+ ret, offset);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Report all data IO complete, in pieces. */
+ btrfs_open_stripe_write_done(cache, fsl, 2 * sectorsize);
+ btrfs_open_stripe_write_done(cache, fsl + 2 * sectorsize,
+ 2 * sectorsize);
+ btrfs_open_stripe_write_done(cache, 2 * fsl, fsl);
+ btrfs_open_stripe_write_done(cache, 3 * fsl, sectorsize);
+
+ /*
+ * Commit-style retirement: closes the open run, returns its tail,
+ * drains (instantly, all IO reported done) and drops the block
+ * group's membership in fs_info->open_stripe_bgs.
+ */
+ btrfs_retire_open_stripes(fs_info);
+ if (!test_check_exists(cache, 3 * fsl + sectorsize,
+ fsl - sectorsize)) {
+ test_err("retired run tail missing from free space");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Only partially filled stripes remain: nothing left to claim. */
+ ret = btrfs_alloc_from_open_stripe(cache, sectorsize, &offset);
+ if (ret != -ENOSPC) {
+ test_err("alloc after retirement returned %d", ret);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Retiring with nothing open is a no-op. */
+ btrfs_retire_open_stripes(fs_info);
+
+ /* Clean up the three partial-stripe remainders. */
+ ret = btrfs_remove_free_space(cache, sectorsize, fsl - sectorsize);
+ if (ret) {
+ test_err("error cleaning up head %d", ret);
+ goto out;
+ }
+ ret = btrfs_remove_free_space(cache, fsl + 4 * sectorsize,
+ fsl - 4 * sectorsize);
+ if (ret) {
+ test_err("error cleaning up tail 1 %d", ret);
+ goto out;
+ }
+ ret = btrfs_remove_free_space(cache, 3 * fsl + sectorsize,
+ fsl - sectorsize);
+ if (ret) {
+ test_err("error cleaning up tail 2 %d", ret);
+ goto out;
+ }
+ ret = 0;
+out:
+ cache->full_stripe_len = orig_fsl;
+ return ret;
+}
+
int btrfs_test_free_space_cache(u32 sectorsize, u32 nodesize)
{
struct btrfs_fs_info *fs_info;
if (ret)
goto out;
ret = test_stripe_claim(cache, sectorsize);
+ if (ret)
+ goto out;
+ ret = test_open_stripe(cache, sectorsize);
out:
btrfs_free_dummy_block_group(cache);
btrfs_free_dummy_root(root);
projects / linux / commitdiff