}
}
+/*
+ * Commit-time recompute of stripe_unusable for every stripe_alloc block group
+ * whose free space changed this transaction. Trapped free space only settles
+ * at commit -- partially filled stripes are returned by the retire above and
+ * deleted extents are unpinned into the free space cache -- so this is where
+ * the counter can be made authoritative. Called after btrfs_finish_extent_commit
+ * so the free space cache reflects this transaction's frees.
+ */
+void btrfs_scan_stripe_unusable(struct btrfs_fs_info *fs_info, bool force)
+{
+ struct btrfs_space_info *sinfo;
+
+ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+ return;
+
+ list_for_each_entry(sinfo, &fs_info->space_info, list) {
+ u64 total = 0;
+ int raid;
+
+ if (!(sinfo->flags & BTRFS_BLOCK_GROUP_DATA))
+ continue;
+ down_read(&sinfo->groups_sem);
+ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
+ struct btrfs_block_group *bg;
+
+ if (!(btrfs_raid_array[raid].bg_flag &
+ BTRFS_BLOCK_GROUP_RAID56_MASK))
+ continue;
+ list_for_each_entry(bg, &sinfo->block_groups[raid],
+ list) {
+ bool dirty = test_and_clear_bit(
+ BLOCK_GROUP_FLAG_STRIPE_UNUSABLE_DIRTY,
+ &bg->runtime_flags);
+
+ /*
+ * @force rescans every armed group regardless of
+ * the dirty flag (debug: isolates a missed
+ * dirty-marking from a genuine cache difference).
+ */
+ if (force || dirty)
+ btrfs_block_group_rescan_stripe_unusable(bg);
+ /*
+ * Sum the authoritative per-group values into the
+ * space_info total so it cannot drift from repeated
+ * arm/disarm/rescan deltas. Disarmed (read-only)
+ * groups are excluded; their free space is already
+ * accounted as read-only.
+ */
+ if (READ_ONCE(bg->stripe_unusable_ready))
+ total += READ_ONCE(bg->stripe_unusable);
+ }
+ }
+ up_read(&sinfo->groups_sem);
+
+ spin_lock(&sinfo->lock);
+ sinfo->bytes_stripe_unusable = total;
+ spin_unlock(&sinfo->lock);
+ }
+}
+
/*
* With stripe-exclusive allocation, no in-place (nocow/prealloc) write may
* land in a raid56 data block group: it could tear a stripe containing
block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED;
spin_unlock(&block_group->lock);
+ /*
+ * The free space cache is now populated; compute the initial trapped
+ * free space of a stripe_alloc block group and arm incremental
+ * accounting. Nothing until this point counted against
+ * stripe_unusable, so the scan captures the full loaded state.
+ */
+ if (!ret)
+ btrfs_block_group_init_stripe_unusable(block_group);
+
#ifdef CONFIG_BTRFS_DEBUG
if (btrfs_should_fragment_free_space(block_group)) {
u64 bytes_used;
WARN_ON(!list_empty(&block_group->io_list));
spin_unlock(&trans->transaction->dirty_bgs_lock);
+ /*
+ * A removed group has already been marked read-only (which disarmed
+ * stripe_unusable), but disarm defensively before the free space cache
+ * is torn down so the space_info total never keeps a stale contribution.
+ */
+ btrfs_block_group_disarm_stripe_unusable(block_group);
+
btrfs_remove_free_space_cache(block_group);
spin_lock(&block_group->space_info->lock);
out:
spin_unlock(&cache->lock);
spin_unlock(&sinfo->lock);
+ /*
+ * A read-only group's free space (trapped included) is now accounted
+ * as read-only and excluded from statfs; drop its stripe_unusable
+ * contribution so it is not subtracted twice. Idempotent, so repeated
+ * ro increments are harmless.
+ */
+ if (!ret)
+ btrfs_block_group_disarm_stripe_unusable(cache);
if (ret == -ENOSPC && btrfs_test_opt(cache->fs_info, ENOSPC_DEBUG)) {
btrfs_info(cache->fs_info,
"unable to make block group %llu ro", cache->start);
trace_btrfs_add_block_group(info, cache, 0);
btrfs_add_bg_to_space_info(info, cache);
+ /*
+ * Full or empty groups are marked cached above without going through
+ * the caching thread, so arm their stripe_unusable accounting here
+ * (both cases scan to zero trapped space). Groups that still need
+ * caching are armed when caching_thread finishes.
+ */
+ if (cache->cached == BTRFS_CACHE_FINISHED)
+ btrfs_block_group_init_stripe_unusable(cache);
+
set_avail_alloc_bits(info, cache->flags);
if (btrfs_chunk_writeable(info, cache->start)) {
if (cache->used == 0) {
btrfs_link_bg_list(cache, &trans->new_bgs);
btrfs_inc_delayed_refs_rsv_bg_inserts(fs_info);
+ /* A freshly created stripe_alloc group starts fully free (0 trapped). */
+ btrfs_block_group_init_stripe_unusable(cache);
+
set_avail_alloc_bits(fs_info, type);
return cache;
}
{
struct btrfs_space_info *sinfo = cache->space_info;
u64 num_bytes;
+ bool became_rw = false;
BUG_ON(!cache->ro);
cache->zone_unusable - cache->used;
sinfo->bytes_readonly -= num_bytes;
list_del_init(&cache->ro_list);
+ became_rw = true;
}
spin_unlock(&cache->lock);
spin_unlock(&sinfo->lock);
+
+ /*
+ * Back to read-write: re-arm stripe_unusable accounting, rescanning
+ * the current free space (it may have changed while read-only). The
+ * scan re-adds this group's trapped free space to the space_info total.
+ */
+ if (became_rw)
+ btrfs_block_group_init_stripe_unusable(cache);
}
static int update_block_group_item(struct btrfs_trans_handle *trans,
* from violations and skips such groups (sticky, debug only).
*/
BLOCK_GROUP_FLAG_STRIPE_RELOC_USED,
+ /*
+ * Free space changed this transaction, so the group's stripe_unusable
+ * counter must be rescanned at commit (stripe_alloc only).
+ */
+ BLOCK_GROUP_FLAG_STRIPE_UNUSABLE_DIRTY,
};
enum btrfs_caching_type {
/* Private to the commit-time retire walk. */
struct list_head open_stripe_retire_list;
+ /*
+ * Free bytes trapped in partially filled stripes of a stripe_alloc
+ * block group: they are genuine free space (present in the free space
+ * cache and the free space tree, so no on-disk accounting changes) but
+ * the stripe-exclusive allocator cannot hand them out until the whole
+ * stripe frees. Computed once at cache-in and maintained incrementally
+ * as free space is added; both this counter and the ready flag are
+ * protected by free_space_ctl->tree_lock. Only statfs subtracts it
+ * (honest f_bavail); the reservation layer is deliberately unchanged.
+ */
+ u64 stripe_unusable;
+ bool stripe_unusable_ready;
+
/*
* Used for several lists:
*
void btrfs_release_data_reloc_bg(struct btrfs_fs_info *fs_info);
bool btrfs_stripe_alloc_forces_cow(struct btrfs_fs_info *fs_info, u64 bytenr);
bool btrfs_is_stripe_alloc_bg(const struct btrfs_block_group *bg);
+void btrfs_scan_stripe_unusable(struct btrfs_fs_info *fs_info, bool force);
#ifdef CONFIG_BTRFS_DEBUG
void btrfs_stripe_check_write(struct btrfs_fs_info *fs_info,
u64 full_stripe_start, bool sub_stripe);
return 0;
}
+static void stripe_unusable_mark_dirty(struct btrfs_block_group *bg);
+
int btrfs_add_free_space(struct btrfs_block_group *block_group,
u64 bytenr, u64 size)
{
if (btrfs_test_opt(block_group->fs_info, DISCARD_SYNC))
trim_state = BTRFS_TRIM_STATE_TRIMMED;
+ stripe_unusable_mark_dirty(block_group);
return __btrfs_add_free_space(block_group, bytenr, size, trim_state);
}
return ret;
}
+/*
+ * Distribute a free range [a, end) into the per-stripe free-byte accumulator,
+ * splitting at full stripe boundaries. Indexing by stripe makes the result
+ * independent of the order ranges arrive in -- the free space cache can visit
+ * them out of offset order (a bitmap entry spans past a later extent entry) --
+ * and of the extent-vs-bitmap representation.
+ */
+static void stripe_dist(u32 *freep, u64 bg_start, u64 fsl, u64 a, u64 end,
+ u64 *counted)
+{
+ while (a < end) {
+ u64 idx = div64_u64(a - bg_start, fsl);
+ u64 stripe_end = bg_start + (idx + 1) * fsl;
+ u64 c = min(end, stripe_end);
+
+ freep[idx] += c - a;
+ *counted += c - a;
+ a = c;
+ }
+}
+
+/* Number of full stripes in a block group; the size of the freep array. */
+static u64 stripe_unusable_nstripes(const struct btrfs_block_group *bg)
+{
+ u64 fsl = bg->full_stripe_len;
+
+ return div64_u64(bg->length + fsl - 1, fsl);
+}
+
+/*
+ * Total trapped free bytes (free space in partially filled stripes) over the
+ * whole block group -- the authoritative definition of stripe_unusable,
+ * recomputed at cache-in and after each commit that changed the group's free
+ * space. Every free byte is accumulated into a per-stripe array (so a stripe
+ * whose total free is neither zero nor a whole stripe is partially filled, and
+ * its free bytes are trapped). @freep is a zeroed array of stripe_unusable_
+ * nstripes() u32 entries the caller allocates before taking the tree lock (it
+ * can be large); this runs with ctl->tree_lock held.
+ */
+static u64 stripe_unusable_scan(struct btrfs_block_group *bg, u32 *freep,
+ u64 nstripes)
+{
+ struct btrfs_free_space_ctl *ctl = bg->free_space_ctl;
+ const u32 unit = bg->fs_info->sectorsize;
+ const u64 fsl = bg->full_stripe_len;
+ const u64 bg_start = bg->start;
+ u64 counted = 0;
+ u64 trapped = 0;
+ struct rb_node *n;
+ u64 i;
+
+ lockdep_assert_held(&ctl->tree_lock);
+
+ for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) {
+ struct btrfs_free_space *e =
+ rb_entry(n, struct btrfs_free_space, offset_index);
+
+ if (e->bitmap) {
+ unsigned long b = 0;
+
+ while (b < BITS_PER_BITMAP) {
+ unsigned long z;
+
+ b = find_next_bit(e->bitmap, BITS_PER_BITMAP, b);
+ if (b >= BITS_PER_BITMAP)
+ break;
+ z = find_next_zero_bit(e->bitmap,
+ BITS_PER_BITMAP, b);
+ stripe_dist(freep, bg_start, fsl,
+ e->offset + (u64)b * unit,
+ e->offset + (u64)z * unit, &counted);
+ b = z;
+ }
+ } else {
+ stripe_dist(freep, bg_start, fsl, e->offset,
+ e->offset + e->bytes, &counted);
+ }
+ }
+
+ /* Every free byte was distributed to exactly one stripe (debug check). */
+ ASSERT(counted == ctl->free_space,
+ "stripe_unusable_scan counted=%llu != free_space=%llu",
+ counted, ctl->free_space);
+
+ for (i = 0; i < nstripes; i++)
+ if (freep[i] && freep[i] < fsl)
+ trapped += freep[i];
+ return trapped;
+}
+
+/*
+ * Compute a stripe_alloc block group's initial trapped free space once its
+ * free space cache is loaded, and enable incremental accounting from here on.
+ * Idempotent-safe to call on a not-yet-populated (new, empty) block group,
+ * where it simply arms accounting with a zero baseline.
+ */
+void btrfs_block_group_init_stripe_unusable(struct btrfs_block_group *bg)
+{
+ struct btrfs_free_space_ctl *ctl = bg->free_space_ctl;
+ u64 trapped, nstripes;
+ u32 *freep;
+
+ if (!btrfs_is_stripe_alloc_bg(bg))
+ return;
+ if (READ_ONCE(bg->stripe_unusable_ready))
+ return;
+
+ nstripes = stripe_unusable_nstripes(bg);
+ freep = kvcalloc(nstripes, sizeof(*freep), GFP_NOFS);
+ if (!freep)
+ return;
+
+ spin_lock(&ctl->tree_lock);
+ if (bg->stripe_unusable_ready) {
+ spin_unlock(&ctl->tree_lock);
+ kvfree(freep);
+ return;
+ }
+ trapped = stripe_unusable_scan(bg, freep, nstripes);
+ bg->stripe_unusable = trapped;
+ bg->stripe_unusable_ready = true;
+ spin_unlock(&ctl->tree_lock);
+ kvfree(freep);
+
+ if (trapped) {
+ spin_lock(&bg->space_info->lock);
+ btrfs_space_info_update_bytes_stripe_unusable(bg->space_info,
+ trapped);
+ spin_unlock(&bg->space_info->lock);
+ }
+}
+
+/*
+ * Remove a stripe_alloc block group's trapped-space contribution from the
+ * space_info total and stop incremental accounting. Used when the group turns
+ * read-only or is removed: from that point its free space is accounted
+ * elsewhere (bytes_readonly / removal) and statfs already excludes it, so
+ * leaving stripe_unusable armed would double count. Re-arm on the way back to
+ * read-write with btrfs_block_group_init_stripe_unusable(). Idempotent.
+ */
+void btrfs_block_group_disarm_stripe_unusable(struct btrfs_block_group *bg)
+{
+ struct btrfs_free_space_ctl *ctl = bg->free_space_ctl;
+ u64 trapped;
+
+ if (!btrfs_is_stripe_alloc_bg(bg))
+ return;
+
+ spin_lock(&ctl->tree_lock);
+ if (!bg->stripe_unusable_ready) {
+ spin_unlock(&ctl->tree_lock);
+ return;
+ }
+ trapped = bg->stripe_unusable;
+ bg->stripe_unusable_ready = false;
+ spin_unlock(&ctl->tree_lock);
+
+ if (trapped) {
+ spin_lock(&bg->space_info->lock);
+ btrfs_space_info_update_bytes_stripe_unusable(bg->space_info,
+ -(s64)trapped);
+ spin_unlock(&bg->space_info->lock);
+ }
+}
+
+
+/*
+ * Recompute a stripe_alloc block group's trapped free space from a fresh scan
+ * and fold the change into the space_info total. Called at commit for groups
+ * whose free space changed this transaction; trapped space only settles at the
+ * commit that returns partially filled stripes and unpins deleted extents, so
+ * a per-commit rescan is both correct and far cheaper than a running tally.
+ */
+void btrfs_block_group_rescan_stripe_unusable(struct btrfs_block_group *bg)
+{
+ struct btrfs_free_space_ctl *ctl = bg->free_space_ctl;
+ u64 nstripes;
+ u32 *freep;
+
+ if (!btrfs_is_stripe_alloc_bg(bg))
+ return;
+
+ nstripes = stripe_unusable_nstripes(bg);
+ freep = kvcalloc(nstripes, sizeof(*freep), GFP_NOFS);
+ if (!freep)
+ return;
+
+ spin_lock(&ctl->tree_lock);
+ if (bg->stripe_unusable_ready) /* skip if disarmed (read-only) */
+ bg->stripe_unusable = stripe_unusable_scan(bg, freep, nstripes);
+ spin_unlock(&ctl->tree_lock);
+ kvfree(freep);
+ /*
+ * The space_info total is recomputed authoritatively from the sum of
+ * armed groups by btrfs_scan_stripe_unusable() after all rescans, so it
+ * cannot drift; no per-group delta is applied here.
+ */
+}
+
+/*
+ * Flag a stripe_alloc block group's stripe_unusable counter for recomputation
+ * at the next commit, because its free space just changed. Cheap (one bit) so
+ * it can sit on the hot allocation and free paths; the actual scan is deferred
+ * to commit. Reading stripe_unusable_ready without the tree lock is a benign
+ * race: a missed or spurious flag only delays or repeats a rescan.
+ */
+static void stripe_unusable_mark_dirty(struct btrfs_block_group *bg)
+{
+ if (READ_ONCE(bg->stripe_unusable_ready))
+ set_bit(BLOCK_GROUP_FLAG_STRIPE_UNUSABLE_DIRTY, &bg->runtime_flags);
+}
+
/*
* Align an offset up to the next full stripe boundary. Full stripe geometry
* is relative to the start of the block group, which is not necessarily
btrfs_discard_update_discardable(block_group);
spin_unlock(&ctl->tree_lock);
+ if (!ret)
+ stripe_unusable_mark_dirty(block_group);
if (head_len)
__btrfs_add_free_space(block_group, head_start, head_len,
head_trim_state);
u64 *max_extent_size);
int btrfs_claim_free_stripe_run(struct btrfs_block_group *block_group,
u64 want_bytes, u64 *start, u64 *len);
+void btrfs_block_group_init_stripe_unusable(struct btrfs_block_group *block_group);
+void btrfs_block_group_disarm_stripe_unusable(struct btrfs_block_group *block_group);
+void btrfs_block_group_rescan_stripe_unusable(struct btrfs_block_group *block_group);
void btrfs_dump_free_space(struct btrfs_block_group *block_group,
u64 bytes);
int btrfs_find_space_cluster(struct btrfs_block_group *block_group,
return s_info->bytes_used + s_info->bytes_reserved +
s_info->bytes_pinned + s_info->bytes_readonly +
s_info->bytes_zone_unusable +
+ s_info->bytes_stripe_unusable +
(may_use_included ? s_info->bytes_may_use : 0);
}
u64 bytes_readonly; /* total bytes that are read only */
u64 bytes_zone_unusable; /* total bytes that are unusable until
resetting the device zone */
+ u64 bytes_stripe_unusable; /* total free bytes trapped in partially
+ filled stripes of stripe_alloc block
+ groups; free but not allocatable until
+ the whole stripe frees. Counted in
+ btrfs_space_info_used() so reservations
+ cannot admit writes against it, and
+ subtracted in statfs (from a different
+ base -- the free space cache walk). */
u64 max_extent_size; /* This will hold the maximum extent size of
the space info if we had an ENOSPC in the
DECLARE_SPACE_INFO_UPDATE(bytes_may_use, "space_info");
DECLARE_SPACE_INFO_UPDATE(bytes_pinned, "pinned");
DECLARE_SPACE_INFO_UPDATE(bytes_zone_unusable, "zone_unusable");
+DECLARE_SPACE_INFO_UPDATE(bytes_stripe_unusable, "stripe_unusable");
int btrfs_init_space_info(struct btrfs_fs_info *fs_info);
void btrfs_add_bg_to_space_info(struct btrfs_fs_info *info,
factor = btrfs_bg_type_to_factor(
btrfs_raid_array[i].bg_flag);
}
+
+ /*
+ * Free space trapped in partially filled stripes of
+ * stripe_alloc (raid56 write-hole-safe) block groups is
+ * present in the free space tree but cannot be allocated
+ * until the whole stripe frees. Subtract it so f_bavail
+ * reflects what can actually be allocated. Only armed,
+ * non-read-only groups contribute (read-only groups are
+ * already removed above); the counter is logical, so
+ * scale it to the disk units of total_free_data.
+ */
+ if (found->bytes_stripe_unusable) {
+ u64 su = found->bytes_stripe_unusable * factor;
+
+ if (su > total_free_data)
+ su = total_free_data;
+ total_free_data -= su;
+ }
}
/*
}
BTRFS_ATTR_W(space_info, force_chunk_alloc, btrfs_force_chunk_alloc_store);
+/*
+ * Force a full recompute of stripe_unusable across every armed stripe_alloc
+ * block group, ignoring the per-group dirty flag. If the reported value rises
+ * to match a fresh umount/mount scan, groups were being missed by dirty
+ * marking; if it stays put, the live cache genuinely differs from a reload.
+ */
+static ssize_t btrfs_stripe_unusable_rescan_store(struct kobject *kobj,
+ struct kobj_attribute *a,
+ const char *buf, size_t len)
+{
+ struct btrfs_fs_info *fs_info = to_fs_info(get_btrfs_kobj(kobj));
+ bool val;
+ int ret;
+
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+
+ ret = kstrtobool(buf, &val);
+ if (ret)
+ return ret;
+ if (!val)
+ return -EINVAL;
+
+ btrfs_scan_stripe_unusable(fs_info, true);
+ return len;
+}
+BTRFS_ATTR_W(space_info, stripe_unusable_rescan,
+ btrfs_stripe_unusable_rescan_store);
+
#endif
SPACE_INFO_ATTR(flags);
SPACE_INFO_ATTR(bytes_may_use);
SPACE_INFO_ATTR(bytes_readonly);
SPACE_INFO_ATTR(bytes_zone_unusable);
+SPACE_INFO_ATTR(bytes_stripe_unusable);
SPACE_INFO_ATTR(disk_used);
SPACE_INFO_ATTR(disk_total);
SPACE_INFO_ATTR(reclaim_count);
BTRFS_ATTR_PTR(space_info, bytes_may_use),
BTRFS_ATTR_PTR(space_info, bytes_readonly),
BTRFS_ATTR_PTR(space_info, bytes_zone_unusable),
+ BTRFS_ATTR_PTR(space_info, bytes_stripe_unusable),
BTRFS_ATTR_PTR(space_info, disk_used),
BTRFS_ATTR_PTR(space_info, disk_total),
BTRFS_ATTR_PTR(space_info, bg_reclaim_threshold),
BTRFS_ATTR_PTR(space_info, periodic_reclaim),
#ifdef CONFIG_BTRFS_DEBUG
BTRFS_ATTR_PTR(space_info, force_chunk_alloc),
+ BTRFS_ATTR_PTR(space_info, stripe_unusable_rescan),
#endif
NULL,
};
if (ret)
goto scrub_continue;
+ /*
+ * The free space cache now reflects this transaction's frees; recompute
+ * stripe_unusable for stripe_alloc groups whose free space changed.
+ */
+ btrfs_scan_stripe_unusable(fs_info, false);
+
if (test_bit(BTRFS_TRANS_HAVE_FREE_BGS, &cur_trans->flags))
btrfs_clear_space_info_full(fs_info);
TP_ARGS(fs_info, sinfo, old, diff)
);
+DEFINE_EVENT(btrfs__space_info_update, update_bytes_stripe_unusable,
+
+ TP_PROTO(const struct btrfs_fs_info *fs_info,
+ const struct btrfs_space_info *sinfo, u64 old, s64 diff),
+
+ TP_ARGS(fs_info, sinfo, old, diff)
+);
+
DECLARE_EVENT_CLASS(btrfs_raid56_bio,
TP_PROTO(const struct btrfs_raid_bio *rbio,
projects / linux / commitdiff