spin_lock(&ctl->tree_lock);
+ /*
+ * New free space may merge into a claimable stripe run; forget any
+ * cached "no claimable run" verdict (see btrfs_claim_free_stripe_run()).
+ */
+ block_group->max_claimable_run = (u64)-1;
+
if (try_merge_free_space(ctl, info, true))
goto link;
cluster->block_group = NULL;
cluster->window_start = 0;
list_del_init(&cluster->block_group_list);
+ /* As for any free space added back; see __btrfs_add_free_space(). */
+ block_group->max_claimable_run = (u64)-1;
node = rb_first(&cluster->root);
while (node) {
}
/*
- * Claim a contiguous run of fully-free full stripes.
+ * Entry-blind complement to the by-size fast path in
+ * btrfs_claim_free_stripe_run(): walk the aligned stripe grid in
+ * address order, accumulating contiguous free coverage
+ * with point lookups, looking for a run of fully-free full stripes no
+ * single entry contains.
+ *
+ * Address order is the point. Walking the entry tree does not visit
+ * free space in address order: a bitmap entry sorts at its window base
+ * while its set-bit runs extend to the window end, so extent entries
+ * inside the window sort after the bitmap but describe lower addresses.
+ * A coverage accumulator fed by an entry walk breaks exactly at the
+ * bitmap window boundary -- which is where the stripes this path exists
+ * to find live.
+ *
+ * ctl->tree_lock must be held.
+ */
+static bool find_free_stripe_run_slow(struct btrfs_block_group *block_group,
+ u64 want_bytes, u64 *run_start,
+ u64 *run_len)
+{
+ struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
+ const u32 unit = block_group->fs_info->sectorsize;
+ const u64 fsl = block_group->full_stripe_len;
+ const u64 bg_end = block_group->start + block_group->length;
+ u64 cand = stripe_run_align(block_group, block_group->start);
+
+ lockdep_assert_held(&ctl->tree_lock);
+
+ while (cand + fsl <= bg_end) {
+ u64 covered = 0;
+
+ /* Accumulate contiguous free coverage upward from cand. */
+ while (covered < want_bytes) {
+ const u64 off = cand + covered;
+ struct btrfs_free_space *info;
+ u64 piece = 0;
+
+ if (off >= bg_end)
+ break;
+ info = tree_search_offset(ctl, off, 0, 0);
+ if (info && !info->bitmap) {
+ piece = info->offset + info->bytes - off;
+ } else {
+ info = tree_search_offset(ctl,
+ offset_to_bitmap(ctl, off), 1, 0);
+ if (info) {
+ unsigned long bit =
+ div64_u64(off - info->offset, unit);
+ unsigned long zero;
+
+ if (test_bit(bit, info->bitmap)) {
+ zero = find_next_zero_bit(
+ info->bitmap,
+ BITS_PER_BITMAP,
+ bit);
+ piece = (u64)(zero - bit) * unit;
+ }
+ }
+ }
+ if (!piece)
+ break;
+ covered += piece;
+ }
+
+ while (covered >= fsl) {
+ u64 tlen = min(want_bytes,
+ div64_u64(covered, fsl) * fsl);
+
+ if (btrfs_stripe_run_range_usable(block_group, cand,
+ &tlen)) {
+ *run_start = cand;
+ *run_len = tlen;
+ return true;
+ }
+ /* Overlaps a live run; try the next stripe. */
+ cand += fsl;
+ covered -= fsl;
+ }
+ /*
+ * The byte at cand + covered is not free, so no stripe
+ * containing it can be claimed: restart at the first
+ * stripe boundary past it.
+ */
+ cand = stripe_run_align(block_group, cand + covered + unit);
+ }
+ return false;
+}
+
+/*
+ * Remove the fully-free run [start, start + len) found by
+ * find_free_stripe_run_slow() from the free space cache, piece by piece
+ * across the entries covering it, without ever dropping ctl->tree_lock:
+ * the find and the removal form one critical section, so a racing
+ * claimer can never see (and double-claim) a half-removed run. @spare
+ * is a caller-allocated entry for the at-most-one split of an extent
+ * entry the run is strictly inside of; returns true if it was consumed.
+ *
+ * ctl->tree_lock must be held.
+ */
+static bool claim_stripe_run_pieces(struct btrfs_block_group *block_group,
+ u64 start, u64 len,
+ struct btrfs_free_space *spare)
+{
+ struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
+ struct btrfs_discard_ctl *discard_ctl =
+ &block_group->fs_info->discard_ctl;
+ const u32 unit = block_group->fs_info->sectorsize;
+ bool spare_used = false;
+ u64 off = start;
+ u64 remaining = len;
+
+ lockdep_assert_held(&ctl->tree_lock);
+
+ while (remaining) {
+ struct btrfs_free_space *info;
+ u64 piece;
+
+ info = tree_search_offset(ctl, off, 0, 0);
+ if (!info)
+ info = tree_search_offset(ctl,
+ offset_to_bitmap(ctl, off), 1, 0);
+ if (WARN_ON(!info))
+ break;
+
+ if (info->bitmap) {
+ const unsigned long bit =
+ offset_to_bit(info->offset, unit, off);
+ const unsigned long zero =
+ find_next_zero_bit(info->bitmap,
+ BITS_PER_BITMAP, bit);
+
+ /*
+ * Only the set bits from @off: the run may go on in
+ * an extent entry inside this bitmap's window.
+ */
+ piece = min(remaining, (u64)(zero - bit) * unit);
+ if (WARN_ON(!piece))
+ break;
+ if (!btrfs_free_space_trimmed(info))
+ atomic64_add(piece,
+ &discard_ctl->discard_bytes_saved);
+ bitmap_clear_bits(ctl, info, off, piece, true);
+ if (!info->bytes)
+ free_bitmap(ctl, info);
+ } else {
+ const u64 e_end = info->offset + info->bytes;
+ int ret2;
+
+ piece = min(remaining, e_end - off);
+ if (!btrfs_free_space_trimmed(info))
+ atomic64_add(piece,
+ &discard_ctl->discard_bytes_saved);
+ unlink_free_space(ctl, info, true);
+ if (off == info->offset && piece == info->bytes) {
+ kmem_cache_free(btrfs_free_space_cachep, info);
+ } else if (off == info->offset) {
+ info->offset += piece;
+ info->bytes -= piece;
+ ret2 = link_free_space(ctl, info);
+ ASSERT(!ret2); /* -EEXIST; Logic error */
+ } else if (off + piece == e_end) {
+ info->bytes = off - info->offset;
+ ret2 = link_free_space(ctl, info);
+ ASSERT(!ret2); /* -EEXIST; Logic error */
+ } else {
+ /*
+ * Strictly inside: split head off, spare
+ * becomes the tail.
+ */
+ ASSERT(!spare_used);
+ spare->offset = off + piece;
+ spare->bytes = e_end - spare->offset;
+ spare->trim_state = info->trim_state;
+ info->bytes = off - info->offset;
+ ret2 = link_free_space(ctl, info);
+ ASSERT(!ret2); /* -EEXIST; Logic error */
+ ret2 = link_free_space(ctl, spare);
+ ASSERT(!ret2); /* -EEXIST; Logic error */
+ spare_used = true;
+ }
+ }
+ off += piece;
+ remaining -= piece;
+ }
+ return spare_used;
+}
+
+/*
+ * Claim a contiguous run of fully free full stripes.
*
- * On success the run is removed from the free space cache and the caller owns
- * [*start, *start + *len) exclusively until it either allocates from it or
- * returns the unused remainder with btrfs_add_free_space().
+ * On success the run is removed from the free space cache and the caller
+ * owns [*start, *start + *len) until it allocates from it or returns the
+ * remainder with btrfs_add_free_space(). @want_bytes is a maximum; the
+ * smallest claim is one full stripe.
*
- * want_bytes must be a full_stripe_len multiple and is a maximum; the
- * smallest successful claim is a single full stripe. A fully-free stripe
- * split across two free space entries (e.g. an extent entry adjoining a
- * bitmap) is not found: adjacent extent entries merge on insert, but
- * extent/bitmap neighbours do not. This errs toward missing a claimable
- * stripe, never toward claiming a non-free byte.
+ * A fully free stripe may span free space entries: extent and bitmap
+ * entries do not merge, and when full_stripe_len is not a power of two,
+ * stripe boundaries drift through the fixed bitmap windows. The slow path
+ * finds such stripes; stripe_claimable counts them, so missing them would
+ * fail writes that admission accepted.
*
- * Returns 0 on success, -ENOSPC if no aligned fully-free stripe is present
- * in this block group.
+ * Returns 0 on success, -ENOSPC if no fully free stripe is present, or
+ * -ENOMEM if the slow path could not allocate.
*/
int btrfs_claim_free_stripe_run(struct btrfs_block_group *block_group,
u64 want_bytes, u64 *start, u64 *len)
const u64 fsl = block_group->full_stripe_len;
struct btrfs_free_space *entry;
struct rb_node *node;
+ struct btrfs_free_space *spare = NULL;
enum btrfs_trim_state head_trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
+ bool spare_used = false;
u64 head_start = 0;
u64 head_len = 0;
u64 run_start;
/* Claims are always whole stripes; tolerate an unaligned want_bytes. */
want_bytes = div64_u64(want_bytes, fsl) * fsl;
+ /*
+ * For the head of a claimed extent entry, or for a run strictly
+ * inside one, so neither has to be added back after the lock is
+ * dropped, where an allocation failure would lose it.
+ */
+ spare = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS);
+
spin_lock(&ctl->tree_lock);
+ /*
+ * Fail fast when an earlier exhaustive scan found no claimable run
+ * (max_claimable_run == 0; __btrfs_add_free_space() resets it) or less
+ * than one full stripe is free; otherwise every failed claim rescans a
+ * fragmented group under ctl->tree_lock. Trust the bound only once the
+ * group is cached: cache loading links entries without
+ * __btrfs_add_free_space(), so the reset would be missed.
+ */
+ if (btrfs_block_group_done(block_group) &&
+ (block_group->max_claimable_run < fsl ||
+ ctl->free_space < fsl)) {
+ spin_unlock(&ctl->tree_lock);
+ if (spare)
+ kmem_cache_free(btrfs_free_space_cachep, spare);
+ return -ENOSPC;
+ }
+
if (ctl->free_space < fsl)
goto out;
continue;
goto found;
}
+
+ /*
+ * No single entry holds a full stripe; look for one whose
+ * free space spans entry boundaries. The spare entry is
+ * for the belt-and-braces case of a run strictly inside
+ * one extent entry, whose removal splits it in two.
+ */
+ spin_unlock(&ctl->tree_lock);
+ spare = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS);
+ if (!spare)
+ return -ENOMEM;
+ spin_lock(&ctl->tree_lock);
+ if (!find_free_stripe_run_slow(block_group, want_bytes,
+ &run_start, &run_len))
+ goto out;
+ spare_used = claim_stripe_run_pieces(block_group, run_start,
+ run_len, spare);
+ *start = run_start;
+ *len = run_len;
+ ret = 0;
goto out;
found:
} else {
kmem_cache_free(btrfs_free_space_cachep, entry);
}
+ if (head_len && spare) {
+ spare->offset = head_start;
+ spare->bytes = head_len;
+ spare->trim_state = head_trim_state;
+ ret2 = link_free_space(ctl, spare);
+ ASSERT(!ret2); /* -EEXIST; Logic error */
+ spare_used = true;
+ head_len = 0;
+ }
}
*start = run_start;
*len = run_len;
btrfs_discard_update_discardable(block_group);
spin_unlock(&ctl->tree_lock);
+ if (spare && !spare_used)
+ kmem_cache_free(btrfs_free_space_cachep, spare);
if (!ret)
stripe_unusable_mark_dirty(block_group);
if (head_len)