*
* want_bytes must be a full_stripe_len multiple and is a maximum; the
* smallest successful claim is a single full stripe. A fully-free stripe
- * split across two free space entries (e.g. an extent entry adjoining a
- * bitmap) is not found: adjacent extent entries merge on insert, but
- * extent/bitmap neighbours do not. This errs toward missing a claimable
- * stripe, never toward claiming a non-free byte.
+ * split across free space entries (an extent entry adjoining a bitmap, or
+ * two bitmaps: adjacent extent entries merge on insert, but extent/bitmap
+ * neighbours and neighbouring bitmap windows do not) is found by the
+ * entry-blind slow path. It has to be: stripe_unusable accounting is
+ * entry-blind too, so it admits data reservations against exactly these
+ * stripes, and a stripe the accounting admits but the claim cannot reach
+ * becomes a buffered write that fails -ENOSPC at writeback, dropping the
+ * pages -- full_stripe_len is not a power of two, so stripe boundaries
+ * drift through the fixed bitmap windows and such stripes are a
+ * certainty, not a corner case.
*
* Returns 0 on success, -ENOSPC if no aligned fully-free stripe is present
- * in this block group.
+ * in this block group, -ENOMEM if the slow path could not allocate.
*/
/*
* Claim exactly [start, start + len) from the free space cache, failing
return btrfs_remove_free_space(block_group, start, len);
}
+/*
+ * Entry-blind complement to the by-size fast path in
+ * btrfs_claim_free_stripe_run(): walk the aligned stripe grid in
+ * address order, accumulating contiguous free coverage
+ * with point lookups, looking for a run of fully-free full stripes no
+ * single entry contains.
+ *
+ * Address order is the point. Walking the entry tree does not visit
+ * free space in address order: a bitmap entry sorts at its window base
+ * while its set-bit runs extend to the window end, so extent entries
+ * inside the window sort after the bitmap but describe lower addresses.
+ * A coverage accumulator fed by an entry walk breaks exactly at the
+ * bitmap window boundary -- which is where the stripes this path exists
+ * to find live.
+ *
+ * ctl->tree_lock must be held.
+ */
+static bool find_free_stripe_run_slow(struct btrfs_block_group *block_group,
+ u64 want_bytes, u64 *run_start,
+ u64 *run_len)
+{
+ struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
+ const u32 unit = block_group->fs_info->sectorsize;
+ const u64 fsl = block_group->full_stripe_len;
+ const u64 bg_end = block_group->start + block_group->length;
+ u64 cand = stripe_run_align(block_group, block_group->start);
+
+ lockdep_assert_held(&ctl->tree_lock);
+
+ while (cand + fsl <= bg_end) {
+ u64 covered = 0;
+
+ /* Accumulate contiguous free coverage upward from cand. */
+ while (covered < want_bytes) {
+ const u64 off = cand + covered;
+ struct btrfs_free_space *info;
+ u64 piece = 0;
+
+ if (off >= bg_end)
+ break;
+ info = tree_search_offset(ctl, off, 0, 0);
+ if (info && !info->bitmap) {
+ piece = info->offset + info->bytes - off;
+ } else {
+ info = tree_search_offset(ctl,
+ offset_to_bitmap(ctl, off), 1, 0);
+ if (info) {
+ unsigned long bit =
+ div64_u64(off - info->offset, unit);
+ unsigned long zero;
+
+ if (test_bit(bit, info->bitmap)) {
+ zero = find_next_zero_bit(
+ info->bitmap,
+ BITS_PER_BITMAP,
+ bit);
+ piece = (u64)(zero - bit) * unit;
+ }
+ }
+ }
+ if (!piece)
+ break;
+ covered += piece;
+ }
+
+ while (covered >= fsl) {
+ u64 tlen = min(want_bytes,
+ div64_u64(covered, fsl) * fsl);
+
+ if (btrfs_stripe_run_range_usable(block_group, cand,
+ &tlen)) {
+ *run_start = cand;
+ *run_len = tlen;
+ return true;
+ }
+ /* Overlaps a live run; try the next stripe. */
+ cand += fsl;
+ covered -= fsl;
+ }
+ /*
+ * The byte at cand + covered is not free, so no stripe
+ * containing it can be claimed: restart at the first
+ * stripe boundary past it.
+ */
+ cand = stripe_run_align(block_group, cand + covered + unit);
+ }
+ return false;
+}
+
+/*
+ * Remove the fully-free run [start, start + len) found by
+ * find_free_stripe_run_slow() from the free space cache, piece by piece
+ * across the entries covering it, without ever dropping ctl->tree_lock:
+ * the find and the removal form one critical section, so a racing
+ * claimer can never see (and double-claim) a half-removed run. @spare
+ * is a caller-allocated entry for the at-most-one split of an extent
+ * entry the run is strictly inside of; returns true if it was consumed.
+ *
+ * ctl->tree_lock must be held.
+ */
+static bool claim_stripe_run_pieces(struct btrfs_block_group *block_group,
+ u64 start, u64 len,
+ struct btrfs_free_space *spare)
+{
+ struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
+ struct btrfs_discard_ctl *discard_ctl =
+ &block_group->fs_info->discard_ctl;
+ const u32 unit = block_group->fs_info->sectorsize;
+ bool spare_used = false;
+ u64 off = start;
+ u64 remaining = len;
+
+ lockdep_assert_held(&ctl->tree_lock);
+
+ while (remaining) {
+ struct btrfs_free_space *info;
+ u64 piece;
+
+ info = tree_search_offset(ctl, off, 0, 0);
+ if (!info)
+ info = tree_search_offset(ctl,
+ offset_to_bitmap(ctl, off), 1, 0);
+ if (WARN_ON(!info))
+ break;
+
+ if (info->bitmap) {
+ const u64 b_end = info->offset +
+ (u64)BITS_PER_BITMAP * unit;
+
+ piece = min(remaining, b_end - off);
+ if (!btrfs_free_space_trimmed(info))
+ atomic64_add(piece,
+ &discard_ctl->discard_bytes_saved);
+ bitmap_clear_bits(ctl, info, off, piece, true);
+ if (!info->bytes)
+ free_bitmap(ctl, info);
+ } else {
+ const u64 e_end = info->offset + info->bytes;
+ int ret2;
+
+ piece = min(remaining, e_end - off);
+ if (!btrfs_free_space_trimmed(info))
+ atomic64_add(piece,
+ &discard_ctl->discard_bytes_saved);
+ unlink_free_space(ctl, info, true);
+ if (off == info->offset && piece == info->bytes) {
+ kmem_cache_free(btrfs_free_space_cachep, info);
+ } else if (off == info->offset) {
+ info->offset += piece;
+ info->bytes -= piece;
+ ret2 = link_free_space(ctl, info);
+ ASSERT(!ret2); /* -EEXIST; Logic error */
+ } else if (off + piece == e_end) {
+ info->bytes = off - info->offset;
+ ret2 = link_free_space(ctl, info);
+ ASSERT(!ret2); /* -EEXIST; Logic error */
+ } else {
+ /* Strictly inside: split head off, spare
+ * becomes the tail. */
+ ASSERT(!spare_used);
+ spare->offset = off + piece;
+ spare->bytes = e_end - spare->offset;
+ spare->trim_state = info->trim_state;
+ info->bytes = off - info->offset;
+ ret2 = link_free_space(ctl, info);
+ ASSERT(!ret2); /* -EEXIST; Logic error */
+ ret2 = link_free_space(ctl, spare);
+ ASSERT(!ret2); /* -EEXIST; Logic error */
+ spare_used = true;
+ }
+ }
+ off += piece;
+ remaining -= piece;
+ }
+ return spare_used;
+}
+
int btrfs_claim_free_stripe_run(struct btrfs_block_group *block_group,
u64 want_bytes, u64 *start, u64 *len)
{
const u64 fsl = block_group->full_stripe_len;
struct btrfs_free_space *entry;
struct rb_node *node;
+ struct btrfs_free_space *spare = NULL;
enum btrfs_trim_state head_trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
+ bool spare_used = false;
u64 head_start = 0;
u64 head_len = 0;
u64 run_start;
continue;
goto found;
}
+
+ /*
+ * No single entry holds a full stripe; look for one whose
+ * free space spans entry boundaries. The spare entry is
+ * for the belt-and-braces case of a run strictly inside
+ * one extent entry, whose removal splits it in two.
+ */
+ spin_unlock(&ctl->tree_lock);
+ spare = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS);
+ if (!spare)
+ return -ENOMEM;
+ spin_lock(&ctl->tree_lock);
+ if (!find_free_stripe_run_slow(block_group, want_bytes,
+ &run_start, &run_len))
+ goto out;
+ spare_used = claim_stripe_run_pieces(block_group, run_start,
+ run_len, spare);
+ *start = run_start;
+ *len = run_len;
+ ret = 0;
goto out;
found:
btrfs_discard_update_discardable(block_group);
spin_unlock(&ctl->tree_lock);
+ if (spare && !spare_used)
+ kmem_cache_free(btrfs_free_space_cachep, spare);
if (!ret)
stripe_unusable_mark_dirty(block_group);
if (head_len)
projects / linux / commitdiff