]> git.hungrycats.org Git - linux/commitdiff
btrfs: stripe_alloc: claim fully-free stripes that span free space entries
authorZygo Blaxell <ce3g8jdj@umail.furryterror.org>
Sat, 19 Sep 2026 17:17:47 +0000 (13:17 -0400)
committerZygo Blaxell <ce3g8jdj@umail.furryterror.org>
Sun, 8 Nov 2026 23:37:46 +0000 (18:37 -0500)
The claim fast path searches single entries, so a fully-free full
stripe whose free space spans an entry boundary -- an extent entry
adjoining a bitmap, or two neighbouring bitmap windows -- was
unclaimable.  stripe_unusable accounting is entry-blind and counts
exactly those stripes as claimable, so admission reserves data against
them; at writeback the claim finds nothing, cow_file_range() gets
-ENOSPC, and the already-dirtied pages are dropped.  full_stripe_len
is not a power of two, so stripe boundaries drift through the fixed
128M bitmap windows and a straddling stripe is a certainty near full,
not a corner case.

Observed live at the raid6 fill edge (rolling-failure, then isolated
by fill-edge-debug with enospc_debug): two 384K stripes, each
straddling a bitmap window boundary, held the space_info accounting
786432 bytes above what the claim could reach, and every writeback
allocation against that phantom margin failed -ENOSPC while ~991MB of
genuinely trapped free space sat in the cache.  The reservation margin
cannot absorb this: the gap is per straddling stripe, not per
outstanding extent.

Add an entry-blind slow path: one offset-ordered walk accumulating
contiguous free coverage across entry boundaries, and a piecewise
removal that runs in the same tree_lock critical section as the find,
so a racing claimer cannot see a half-removed run.  The fast path is
unchanged and still serves the common case; the slow path runs only
after it fails, which is the near-full case where a stranded stripe
matters most.

A bitmap piece of a split run is taken only to the end of the set bits
that start at it, so that an extent entry starting inside the same bitmap
window is found and removed as the next piece.  A self-test covers that
layout.

On a large aged filesystem (85 TiB, mostly legacy data, heavily fragmented
free space) the slow path walked the whole group for every small
allocation and found nothing, at 100% CPU in the delalloc workers.  So a
failed exhaustive scan is remembered: bg->max_claimable_run is an upper
bound learned from the scans ((u64)-1 unknown, 0 after a scan proved no
claimable run), claims fail in O(1) while it is below one full stripe or
the group's free space is, and adding free space (frees, unpins, stripe
run returns, and a cluster returning its entries) resets it.  It is only
set and trusted once the group is fully cached, since cache loading links
entries without the reset.

When a claim splits one extent entry, the entry keeps the tail and the
head is linked back with a spare entry allocated before ctl->tree_lock is
taken, so a failed allocation cannot lose it from the cache; without a
spare it is added after the lock as before.

Assisted-by: Claude:claude-fable-5
Assisted-by: Claude:claude-opus-5-5
fs/btrfs/block-group.c
fs/btrfs/block-group.h
fs/btrfs/free-space-cache.c
fs/btrfs/tests/free-space-tests.c

index b9f134df9856f54e0cf0ced161e742794954bbaf..0b420806e298600a42ef81c6d655bfe3dcfea102 100644 (file)
@@ -3684,6 +3684,7 @@ static struct btrfs_block_group *btrfs_create_block_group(
 
        cache->fs_info = fs_info;
        cache->full_stripe_len = btrfs_full_stripe_len(fs_info, start);
+       cache->max_claimable_run = (u64)-1;
 
        cache->discard_index = BTRFS_DISCARD_INDEX_UNUSED;
 
index afc11c2249ac5f2745e46148f1d010de1c92d3d7..8011c070891e7d09275a526b4e8531cce0c11b43 100644 (file)
@@ -278,6 +278,18 @@ struct btrfs_block_group {
         * free_space_ctl->tree_lock and space_info->lock.
         */
        u64 stripe_unusable;
+       /*
+        * Upper bound on the longest run btrfs_claim_free_stripe_run() can
+        * find, learned from actual scans: (u64)-1 means unknown, 0 means an
+        * exhaustive scan proved there is no claimable run.  Claims fail in
+        * O(1) while the bound is below one full stripe; adding free space
+        * resets it to unknown.  Unlike stripe_claimable (an accounting
+        * estimate), this caches what the scan itself proved, so it does not
+        * depend on stripe_claimable being exact.  Protected by
+        * free_space_ctl->tree_lock; only meaningful once the block group is
+        * fully cached.
+        */
+       u64 max_claimable_run;
        bool stripe_unusable_ready;
 
        /*
index 40c6af1c55ffe17985b538a982e36eb24bc21e7c..6238392424d72ec90755ed310d1f523b9a3bf60e 100644 (file)
@@ -2641,6 +2641,12 @@ static int __btrfs_add_free_space(struct btrfs_block_group *block_group,
 
        spin_lock(&ctl->tree_lock);
 
+       /*
+        * New free space may merge into a claimable stripe run; forget any
+        * cached "no claimable run" verdict (see btrfs_claim_free_stripe_run()).
+        */
+       block_group->max_claimable_run = (u64)-1;
+
        if (try_merge_free_space(ctl, info, true))
                goto link;
 
@@ -3029,6 +3035,8 @@ static void __btrfs_return_cluster_to_free_space(
        cluster->block_group = NULL;
        cluster->window_start = 0;
        list_del_init(&cluster->block_group_list);
+       /* As for any free space added back; see __btrfs_add_free_space(). */
+       block_group->max_claimable_run = (u64)-1;
 
        node = rb_first(&cluster->root);
        while (node) {
@@ -3475,21 +3483,209 @@ static bool find_stripe_run_in_bitmap(struct btrfs_free_space_ctl *ctl,
 }
 
 /*
- * Claim a contiguous run of fully-free full stripes.
+ * Entry-blind complement to the by-size fast path in
+ * btrfs_claim_free_stripe_run(): walk the aligned stripe grid in
+ * address order, accumulating contiguous free coverage
+ * with point lookups, looking for a run of fully-free full stripes no
+ * single entry contains.
+ *
+ * Address order is the point.  Walking the entry tree does not visit
+ * free space in address order: a bitmap entry sorts at its window base
+ * while its set-bit runs extend to the window end, so extent entries
+ * inside the window sort after the bitmap but describe lower addresses.
+ * A coverage accumulator fed by an entry walk breaks exactly at the
+ * bitmap window boundary -- which is where the stripes this path exists
+ * to find live.
+ *
+ * ctl->tree_lock must be held.
+ */
+static bool find_free_stripe_run_slow(struct btrfs_block_group *block_group,
+                                     u64 want_bytes, u64 *run_start,
+                                     u64 *run_len)
+{
+       struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
+       const u32 unit = block_group->fs_info->sectorsize;
+       const u64 fsl = block_group->full_stripe_len;
+       const u64 bg_end = block_group->start + block_group->length;
+       u64 cand = stripe_run_align(block_group, block_group->start);
+
+       lockdep_assert_held(&ctl->tree_lock);
+
+       while (cand + fsl <= bg_end) {
+               u64 covered = 0;
+
+               /* Accumulate contiguous free coverage upward from cand. */
+               while (covered < want_bytes) {
+                       const u64 off = cand + covered;
+                       struct btrfs_free_space *info;
+                       u64 piece = 0;
+
+                       if (off >= bg_end)
+                               break;
+                       info = tree_search_offset(ctl, off, 0, 0);
+                       if (info && !info->bitmap) {
+                               piece = info->offset + info->bytes - off;
+                       } else {
+                               info = tree_search_offset(ctl,
+                                               offset_to_bitmap(ctl, off), 1, 0);
+                               if (info) {
+                                       unsigned long bit =
+                                               div64_u64(off - info->offset, unit);
+                                       unsigned long zero;
+
+                                       if (test_bit(bit, info->bitmap)) {
+                                               zero = find_next_zero_bit(
+                                                               info->bitmap,
+                                                               BITS_PER_BITMAP,
+                                                               bit);
+                                               piece = (u64)(zero - bit) * unit;
+                                       }
+                               }
+                       }
+                       if (!piece)
+                               break;
+                       covered += piece;
+               }
+
+               while (covered >= fsl) {
+                       u64 tlen = min(want_bytes,
+                                      div64_u64(covered, fsl) * fsl);
+
+                       if (btrfs_stripe_run_range_usable(block_group, cand,
+                                                         &tlen)) {
+                               *run_start = cand;
+                               *run_len = tlen;
+                               return true;
+                       }
+                       /* Overlaps a live run; try the next stripe. */
+                       cand += fsl;
+                       covered -= fsl;
+               }
+               /*
+                * The byte at cand + covered is not free, so no stripe
+                * containing it can be claimed: restart at the first
+                * stripe boundary past it.
+                */
+               cand = stripe_run_align(block_group, cand + covered + unit);
+       }
+       return false;
+}
+
+/*
+ * Remove the fully-free run [start, start + len) found by
+ * find_free_stripe_run_slow() from the free space cache, piece by piece
+ * across the entries covering it, without ever dropping ctl->tree_lock:
+ * the find and the removal form one critical section, so a racing
+ * claimer can never see (and double-claim) a half-removed run.  @spare
+ * is a caller-allocated entry for the at-most-one split of an extent
+ * entry the run is strictly inside of; returns true if it was consumed.
+ *
+ * ctl->tree_lock must be held.
+ */
+static bool claim_stripe_run_pieces(struct btrfs_block_group *block_group,
+                                   u64 start, u64 len,
+                                   struct btrfs_free_space *spare)
+{
+       struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
+       struct btrfs_discard_ctl *discard_ctl =
+                                       &block_group->fs_info->discard_ctl;
+       const u32 unit = block_group->fs_info->sectorsize;
+       bool spare_used = false;
+       u64 off = start;
+       u64 remaining = len;
+
+       lockdep_assert_held(&ctl->tree_lock);
+
+       while (remaining) {
+               struct btrfs_free_space *info;
+               u64 piece;
+
+               info = tree_search_offset(ctl, off, 0, 0);
+               if (!info)
+                       info = tree_search_offset(ctl,
+                                       offset_to_bitmap(ctl, off), 1, 0);
+               if (WARN_ON(!info))
+                       break;
+
+               if (info->bitmap) {
+                       const unsigned long bit =
+                               offset_to_bit(info->offset, unit, off);
+                       const unsigned long zero =
+                               find_next_zero_bit(info->bitmap,
+                                                  BITS_PER_BITMAP, bit);
+
+                       /*
+                        * Only the set bits from @off: the run may go on in
+                        * an extent entry inside this bitmap's window.
+                        */
+                       piece = min(remaining, (u64)(zero - bit) * unit);
+                       if (WARN_ON(!piece))
+                               break;
+                       if (!btrfs_free_space_trimmed(info))
+                               atomic64_add(piece,
+                                            &discard_ctl->discard_bytes_saved);
+                       bitmap_clear_bits(ctl, info, off, piece, true);
+                       if (!info->bytes)
+                               free_bitmap(ctl, info);
+               } else {
+                       const u64 e_end = info->offset + info->bytes;
+                       int ret2;
+
+                       piece = min(remaining, e_end - off);
+                       if (!btrfs_free_space_trimmed(info))
+                               atomic64_add(piece,
+                                            &discard_ctl->discard_bytes_saved);
+                       unlink_free_space(ctl, info, true);
+                       if (off == info->offset && piece == info->bytes) {
+                               kmem_cache_free(btrfs_free_space_cachep, info);
+                       } else if (off == info->offset) {
+                               info->offset += piece;
+                               info->bytes -= piece;
+                               ret2 = link_free_space(ctl, info);
+                               ASSERT(!ret2); /* -EEXIST; Logic error */
+                       } else if (off + piece == e_end) {
+                               info->bytes = off - info->offset;
+                               ret2 = link_free_space(ctl, info);
+                               ASSERT(!ret2); /* -EEXIST; Logic error */
+                       } else {
+                               /*
+                                * Strictly inside: split head off, spare
+                                * becomes the tail.
+                                */
+                               ASSERT(!spare_used);
+                               spare->offset = off + piece;
+                               spare->bytes = e_end - spare->offset;
+                               spare->trim_state = info->trim_state;
+                               info->bytes = off - info->offset;
+                               ret2 = link_free_space(ctl, info);
+                               ASSERT(!ret2); /* -EEXIST; Logic error */
+                               ret2 = link_free_space(ctl, spare);
+                               ASSERT(!ret2); /* -EEXIST; Logic error */
+                               spare_used = true;
+                       }
+               }
+               off += piece;
+               remaining -= piece;
+       }
+       return spare_used;
+}
+
+/*
+ * Claim a contiguous run of fully free full stripes.
  *
- * On success the run is removed from the free space cache and the caller owns
- * [*start, *start + *len) exclusively until it either allocates from it or
- * returns the unused remainder with btrfs_add_free_space().
+ * On success the run is removed from the free space cache and the caller
+ * owns [*start, *start + *len) until it allocates from it or returns the
+ * remainder with btrfs_add_free_space().  @want_bytes is a maximum; the
+ * smallest claim is one full stripe.
  *
- * want_bytes must be a full_stripe_len multiple and is a maximum; the
- * smallest successful claim is a single full stripe.  A fully-free stripe
- * split across two free space entries (e.g. an extent entry adjoining a
- * bitmap) is not found: adjacent extent entries merge on insert, but
- * extent/bitmap neighbours do not.  This errs toward missing a claimable
- * stripe, never toward claiming a non-free byte.
+ * A fully free stripe may span free space entries: extent and bitmap
+ * entries do not merge, and when full_stripe_len is not a power of two,
+ * stripe boundaries drift through the fixed bitmap windows.  The slow path
+ * finds such stripes; stripe_claimable counts them, so missing them would
+ * fail writes that admission accepted.
  *
- * Returns 0 on success, -ENOSPC if no aligned fully-free stripe is present
- * in this block group.
+ * Returns 0 on success, -ENOSPC if no fully free stripe is present, or
+ * -ENOMEM if the slow path could not allocate.
  */
 int btrfs_claim_free_stripe_run(struct btrfs_block_group *block_group,
                                u64 want_bytes, u64 *start, u64 *len)
@@ -3500,7 +3696,9 @@ int btrfs_claim_free_stripe_run(struct btrfs_block_group *block_group,
        const u64 fsl = block_group->full_stripe_len;
        struct btrfs_free_space *entry;
        struct rb_node *node;
+       struct btrfs_free_space *spare = NULL;
        enum btrfs_trim_state head_trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
+       bool spare_used = false;
        u64 head_start = 0;
        u64 head_len = 0;
        u64 run_start;
@@ -3512,8 +3710,32 @@ int btrfs_claim_free_stripe_run(struct btrfs_block_group *block_group,
        /* Claims are always whole stripes; tolerate an unaligned want_bytes. */
        want_bytes = div64_u64(want_bytes, fsl) * fsl;
 
+       /*
+        * For the head of a claimed extent entry, or for a run strictly
+        * inside one, so neither has to be added back after the lock is
+        * dropped, where an allocation failure would lose it.
+        */
+       spare = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS);
+
        spin_lock(&ctl->tree_lock);
 
+       /*
+        * Fail fast when an earlier exhaustive scan found no claimable run
+        * (max_claimable_run == 0; __btrfs_add_free_space() resets it) or less
+        * than one full stripe is free; otherwise every failed claim rescans a
+        * fragmented group under ctl->tree_lock.  Trust the bound only once the
+        * group is cached: cache loading links entries without
+        * __btrfs_add_free_space(), so the reset would be missed.
+        */
+       if (btrfs_block_group_done(block_group) &&
+           (block_group->max_claimable_run < fsl ||
+            ctl->free_space < fsl)) {
+               spin_unlock(&ctl->tree_lock);
+               if (spare)
+                       kmem_cache_free(btrfs_free_space_cachep, spare);
+               return -ENOSPC;
+       }
+
        if (ctl->free_space < fsl)
                goto out;
 
@@ -3553,6 +3775,26 @@ int btrfs_claim_free_stripe_run(struct btrfs_block_group *block_group,
                        continue;
                goto found;
        }
+
+       /*
+        * No single entry holds a full stripe; look for one whose
+        * free space spans entry boundaries.  The spare entry is
+        * for the belt-and-braces case of a run strictly inside
+        * one extent entry, whose removal splits it in two.
+        */
+       spin_unlock(&ctl->tree_lock);
+       spare = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS);
+       if (!spare)
+               return -ENOMEM;
+       spin_lock(&ctl->tree_lock);
+       if (!find_free_stripe_run_slow(block_group, want_bytes,
+                                      &run_start, &run_len))
+               goto out;
+       spare_used = claim_stripe_run_pieces(block_group, run_start,
+                                            run_len, spare);
+       *start = run_start;
+       *len = run_len;
+       ret = 0;
        goto out;
 
 found:
@@ -3579,6 +3821,15 @@ found:
                } else {
                        kmem_cache_free(btrfs_free_space_cachep, entry);
                }
+               if (head_len && spare) {
+                       spare->offset = head_start;
+                       spare->bytes = head_len;
+                       spare->trim_state = head_trim_state;
+                       ret2 = link_free_space(ctl, spare);
+                       ASSERT(!ret2); /* -EEXIST; Logic error */
+                       spare_used = true;
+                       head_len = 0;
+               }
        }
        *start = run_start;
        *len = run_len;
@@ -3587,6 +3838,8 @@ out:
        btrfs_discard_update_discardable(block_group);
        spin_unlock(&ctl->tree_lock);
 
+       if (spare && !spare_used)
+               kmem_cache_free(btrfs_free_space_cachep, spare);
        if (!ret)
                stripe_unusable_mark_dirty(block_group);
        if (head_len)
index 1ee9632d7e0b6475c64a806bc6bd99abd42a5810..4022cadcd9e4c750a8541c27a74cf67d76e2ee5b 100644 (file)
@@ -1166,6 +1166,33 @@ static int test_stripe_claim(struct btrfs_block_group *cache, u32 sectorsize)
                goto out;
        }
 
+       /*
+        * A free stripe split inside one bitmap window: set bits, then an
+        * extent entry.  The claim must remove both pieces, and nothing else.
+        */
+       ret = test_add_free_space_entry(cache, 0, fsl / 2, 1);
+       if (ret) {
+               test_err("error adding bitmap entry %d", ret);
+               goto out;
+       }
+       ret = test_add_free_space_entry(cache, fsl / 2, fsl / 2, 0);
+       if (ret) {
+               test_err("error adding extent entry %d", ret);
+               goto out;
+       }
+       ret = btrfs_claim_free_stripe_run(cache, fsl, &start, &len);
+       if (ret || start != 0 || len != fsl) {
+               test_err("bitmap+extent claim wrong: ret %d start %llu len %llu",
+                        ret, start, len);
+               ret = -EINVAL;
+               goto out;
+       }
+       ret = check_cache_empty(cache);
+       if (ret) {
+               test_err("bitmap+extent claim left free space behind");
+               goto out;
+       }
+
        /* Non-power-of-two full stripe length (e.g. 3 data stripes). */
        cache->full_stripe_len = fsl_np2;
        ret = btrfs_add_free_space(cache, sectorsize, 4 * fsl_np2);