}
}
-static int __btrfs_add_free_space(struct btrfs_block_group *block_group,
- u64 offset, u64 bytes,
- enum btrfs_trim_state trim_state)
+static void stripe_claimable_mod(struct btrfs_block_group *bg, s64 delta);
+static u64 stripe_touched_bytes(const struct btrfs_block_group *bg,
+ u64 bytenr, u64 size);
+static void stripe_trim_claimable(struct btrfs_block_group *bg, u64 bytenr,
+ u64 size, bool out);
+
+static int add_free_space(struct btrfs_block_group *block_group,
+ u64 offset, u64 bytes,
+ enum btrfs_trim_state trim_state, bool trim_return)
{
struct btrfs_fs_info *fs_info = block_group->fs_info;
struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
if (ret)
kmem_cache_free(btrfs_free_space_cachep, info);
out:
+ /*
+ * Adding free space never credits stripe_claimable: some add paths
+ * re-add space that was never debited (the tail of
+ * btrfs_remove_free_space()'s middle split), and crediting would
+ * count it twice. The counter may only err low (an early ENOSPC); an
+ * over-count admits reservations writeback cannot place. The commit
+ * rescan is the only upward correction -- except for a trim's return,
+ * whose range was debited when the trim took it out (see
+ * stripe_trim_claimable()).
+ */
+ if (!ret && trim_return)
+ stripe_trim_claimable(block_group, offset, bytes, false);
btrfs_discard_update_discardable(block_group);
spin_unlock(&ctl->tree_lock);
return ret;
}
+static int __btrfs_add_free_space(struct btrfs_block_group *block_group,
+ u64 offset, u64 bytes,
+ enum btrfs_trim_state trim_state)
+{
+ return add_free_space(block_group, offset, bytes, trim_state, false);
+}
+
static int __btrfs_add_free_space_zoned(struct btrfs_block_group *block_group,
u64 bytenr, u64 size, bool used)
{
static void stripe_unusable_mark_dirty(struct btrfs_block_group *bg);
+/*
+ * Debit the claimable whole-stripe supply for claims and removals. Returned
+ * free space is never credited here (see add_free_space()), save a trim's
+ * return (stripe_trim_claimable()); the commit rescan restores it. Caller
+ * holds ctl->tree_lock.
+ */
+static void stripe_claimable_mod(struct btrfs_block_group *bg, s64 delta)
+{
+ struct btrfs_space_info *sinfo = bg->space_info;
+
+ if (!READ_ONCE(bg->stripe_unusable_ready) || !delta)
+ return;
+ if (delta < 0 && bg->stripe_claimable < (u64)-delta)
+ delta = -(s64)bg->stripe_claimable;
+ if (!delta)
+ return;
+ bg->stripe_claimable += delta;
+ if (!sinfo)
+ return;
+ spin_lock(&sinfo->lock);
+ if (delta < 0 && sinfo->bytes_stripe_claimable < (u64)-delta)
+ sinfo->bytes_stripe_claimable = 0;
+ else
+ btrfs_space_info_update_bytes_stripe_claimable(sinfo, delta);
+ spin_unlock(&sinfo->lock);
+}
+
+/* The whole-stripe span [bytenr, bytenr + size) touches at all. */
+static u64 stripe_touched_bytes(const struct btrfs_block_group *bg,
+ u64 bytenr, u64 size)
+{
+ const u64 fsl = bg->full_stripe_len;
+ const u64 rel = bytenr - bg->start;
+ const u64 astart = div64_u64(rel, fsl) * fsl;
+ const u64 aend = div64_u64(rel + size + fsl - 1, fsl) * fsl;
+
+ return aend - astart;
+}
+
+/* Is all of [start, end) free? Caller holds ctl->tree_lock. */
+static bool stripe_range_free(struct btrfs_free_space_ctl *ctl,
+ u64 start, u64 end)
+{
+ const u32 unit = ctl->block_group->fs_info->sectorsize;
+ u64 pos = start;
+
+ while (pos < end) {
+ struct btrfs_free_space *e = tree_search_offset(ctl, pos, 0, 0);
+ unsigned long bit;
+
+ /*
+ * An extent entry that starts inside a bitmap's window and
+ * ends before pos hides that bitmap from the search above;
+ * look it up by its own offset, as btrfs_remove_free_space()
+ * does.
+ */
+ if (!e)
+ e = tree_search_offset(ctl, offset_to_bitmap(ctl, pos),
+ 1, 0);
+ if (!e)
+ return false;
+ if (!e->bitmap) {
+ /* An extent entry returned here covers pos. */
+ pos = e->offset + e->bytes;
+ continue;
+ }
+ bit = div_u64(pos - e->offset, unit);
+ if (!test_bit(bit, e->bitmap))
+ return false;
+ pos = e->offset + (u64)find_next_zero_bit(e->bitmap,
+ BITS_PER_BITMAP, bit) * unit;
+ }
+ return true;
+}
+
+/*
+ * The claimable stripes that [bytenr, bytenr + size) belongs to, counting
+ * the range itself as free: by the scan's rule (stripe_unusable_scan()),
+ * every stripe that is wholly free and not inside a live stripe run.
+ * Caller holds ctl->tree_lock; stripe_run_lock nests inside it.
+ */
+static u64 stripe_claimable_with_range(struct btrfs_block_group *bg,
+ u64 bytenr, u64 size)
+{
+ struct btrfs_free_space_ctl *ctl = bg->free_space_ctl;
+ const u64 fsl = bg->full_stripe_len;
+ const u64 end = bytenr + size;
+ const u64 bg_end = btrfs_block_group_end(bg);
+ u64 whole = 0;
+ u64 s;
+
+ for (s = bg->start + div64_u64(bytenr - bg->start, fsl) * fsl;
+ s < end && s + fsl <= bg_end; s += fsl) {
+ if (s < bytenr && !stripe_range_free(ctl, s, bytenr))
+ continue;
+ if (s + fsl > end && !stripe_range_free(ctl, end, s + fsl))
+ continue;
+ if (!btrfs_stripe_run_strands(bg, s))
+ whole += fsl;
+ }
+ return whole;
+}
+
+/*
+ * A trim takes a free range out of the tree for the length of its discard
+ * (trim_no_bitmap(), trim_bitmaps()) and do_trimming() adds it back. A
+ * rescan in between scans a tree without the range, so the incremental
+ * count must not include it either. Each step changes exactly the stripes
+ * the range belongs to: taking it out (@out) makes every one of them that
+ * was claimable unclaimable, and adding it back makes claimable every one
+ * that is claimable now. Debit and credit those, in the tree_lock section
+ * that unlinks or links the range, so the count stays as exact as it was.
+ * Caller holds ctl->tree_lock; for @out, the range is already unlinked.
+ */
+static void stripe_trim_claimable(struct btrfs_block_group *bg, u64 bytenr,
+ u64 size, bool out)
+{
+ s64 delta;
+
+ if (!btrfs_is_stripe_alloc_bg(bg))
+ return;
+ if (!READ_ONCE(bg->stripe_unusable_ready) || !size)
+ return;
+ delta = stripe_claimable_with_range(bg, bytenr, size);
+ stripe_claimable_mod(bg, out ? -delta : delta);
+}
+
/*
* Count the sub-stripe head and tail of a returning free range as trapped
* at once: reservations read bytes_stripe_unusable mid-transaction, and the
spin_lock(&ctl->tree_lock);
+ /*
+ * The removed bytes leave free space; every whole stripe the range
+ * touches may lose wholeness (a boundary fragment removal breaks a
+ * fully free stripe as surely as an interior one empties it).
+ * Rounding the decrement OUT to touched stripes errs claimable-low.
+ * The trapped counter is deliberately not decremented for removed
+ * fragments: leaving it high is the safe direction, and the rescan
+ * settles both.
+ */
+ if (btrfs_is_stripe_alloc_bg(block_group))
+ stripe_claimable_mod(block_group,
+ -(s64)stripe_touched_bytes(block_group, offset, bytes));
+
again:
ret = 0;
if (!bytes)
void btrfs_block_group_init_stripe_unusable(struct btrfs_block_group *bg)
{
struct btrfs_free_space_ctl *ctl = bg->free_space_ctl;
- u64 trapped, nstripes;
+ u64 trapped, claimable, nstripes;
u32 *freep;
if (!btrfs_is_stripe_alloc_bg(bg))
}
trapped = stripe_unusable_scan(bg, freep, nstripes);
bg->stripe_unusable = trapped;
+ /* Every free byte is claimable or trapped; the scan split them. */
+ bg->stripe_claimable = ctl->free_space - trapped;
+ claimable = bg->stripe_claimable;
bg->stripe_unusable_ready = true;
spin_unlock(&ctl->tree_lock);
kvfree(freep);
- if (trapped) {
+ if (trapped || claimable) {
spin_lock(&bg->space_info->lock);
btrfs_space_info_update_bytes_stripe_unusable(bg->space_info,
trapped);
+ btrfs_space_info_update_bytes_stripe_claimable(bg->space_info,
+ claimable);
spin_unlock(&bg->space_info->lock);
}
}
{
struct btrfs_free_space_ctl *ctl = bg->free_space_ctl;
u64 trapped;
+ u64 claimable;
if (!btrfs_is_stripe_alloc_bg(bg))
return;
return;
}
trapped = bg->stripe_unusable;
+ claimable = bg->stripe_claimable;
+ bg->stripe_claimable = 0;
bg->stripe_unusable_ready = false;
spin_unlock(&ctl->tree_lock);
- if (trapped) {
+ if (trapped || claimable) {
spin_lock(&bg->space_info->lock);
btrfs_space_info_update_bytes_stripe_unusable(bg->space_info,
-(s64)trapped);
+ btrfs_space_info_update_bytes_stripe_claimable(bg->space_info,
+ -(s64)claimable);
spin_unlock(&bg->space_info->lock);
}
}
stripe_return_clusters(bg);
spin_lock(&ctl->tree_lock);
- if (bg->stripe_unusable_ready) /* skip if disarmed (read-only) */
+ if (bg->stripe_unusable_ready) { /* skip if disarmed (read-only) */
+ u64 scanned;
+
bg->stripe_unusable = stripe_unusable_scan(bg, freep, nstripes);
+ scanned = ctl->free_space - bg->stripe_unusable;
+ /*
+ * The incremental rules may only under-count claimable
+ * supply; more than the scan finds means a missed debit,
+ * which admits reservations writeback cannot honor. Rate
+ * limited, not _ONCE, because such a gap recurs every commit.
+ * The assignment below keeps the rescan authoritative.
+ */
+ WARN_RATELIMIT(bg->stripe_claimable > scanned,
+ "bg %llu incremental stripe_claimable %llu above scanned %llu",
+ bg->start, bg->stripe_claimable, scanned);
+ bg->stripe_claimable = scanned;
+ }
spin_unlock(&ctl->tree_lock);
kvfree(freep);
/*
}
spare_used = claim_stripe_run_pieces(block_group, run_start,
run_len, spare);
+ stripe_claimable_mod(block_group, -(s64)run_len);
*start = run_start;
*len = run_len;
ret = 0;
head_len = 0;
}
}
+ stripe_claimable_mod(block_group, -(s64)run_len);
*start = run_start;
*len = run_len;
ret = 0;
mutex_lock(&ctl->cache_writeout_mutex);
if (reserved_start < start)
- __btrfs_add_free_space(block_group, reserved_start,
- start - reserved_start,
- reserved_trim_state);
+ add_free_space(block_group, reserved_start,
+ start - reserved_start, reserved_trim_state, true);
if (end < reserved_end)
- __btrfs_add_free_space(block_group, end, reserved_end - end,
- reserved_trim_state);
- __btrfs_add_free_space(block_group, start, bytes, trim_state);
+ add_free_space(block_group, end, reserved_end - end,
+ reserved_trim_state, true);
+ add_free_space(block_group, start, bytes, trim_state, true);
list_del(&trim_entry->list);
mutex_unlock(&ctl->cache_writeout_mutex);
unlink_free_space(ctl, entry, true);
kmem_cache_free(btrfs_free_space_cachep, entry);
}
+ stripe_trim_claimable(block_group, extent_start, extent_bytes,
+ true);
spin_unlock(&ctl->tree_lock);
trim_entry.start = extent_start;
bitmap_clear_bits(ctl, entry, start, bytes, true);
if (entry->bytes == 0)
free_bitmap(ctl, entry);
+ stripe_trim_claimable(block_group, start, bytes, true);
spin_unlock(&ctl->tree_lock);
trim_entry.start = start;