spin_lock(&sinfo->lock);
sinfo->bytes_stripe_unusable = total;
+ /*
+ * The rescan usually lowers the counter (mid-transaction
+ * incremental adds overcount conservatively); admission
+ * tickets may be waiting on exactly that headroom.
+ */
+ btrfs_try_granting_tickets(sinfo);
spin_unlock(&sinfo->lock);
}
}
cache_size *= fs_info->sectorsize;
ret = btrfs_check_data_free_space(BTRFS_I(inode), &data_reserved, 0,
- cache_size, false);
+ cache_size, false, NULL);
if (ret)
goto out_put;
return btrfs_ino(inode) != BTRFS_BTREE_INODE_OBJECTID;
}
+void btrfs_stripe_margin_mod(struct btrfs_inode *inode, int mod);
+
static inline void btrfs_mod_outstanding_extents(struct btrfs_inode *inode,
int mod)
{
return;
trace_btrfs_inode_mod_outstanding_extents(inode->root, btrfs_ino(inode),
mod, inode->outstanding_extents);
+ btrfs_stripe_margin_mod(inode, mod);
}
/*
int btrfs_check_data_free_space(struct btrfs_inode *inode,
struct extent_changeset **reserved, u64 start,
- u64 len, bool noflush)
+ u64 len, bool noflush, u64 *stripe_held)
{
struct btrfs_fs_info *fs_info = inode->root->fs_info;
enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_DATA;
+ u64 stripe_margin = 0;
int ret;
/* align the range */
else if (btrfs_is_free_space_inode(inode))
flush = BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE;
- ret = btrfs_reserve_data_bytes(data_sinfo_for_inode(inode), len, flush);
+ /*
+ * stripe_alloc: reserve the pessimistic margin on top of the bytes, so
+ * the flush below must produce enough claimable whole stripes for this
+ * write's worst case, and a write that cannot be covered gets ENOSPC
+ * here rather than being dropped at writeback. The metadata
+ * reservation re-charges the margin into bytes_stripe_margin
+ * (btrfs_mod_outstanding_extents()). With @stripe_held the probe stays
+ * in bytes_may_use until then (btrfs_release_data_margin_probe()), so
+ * no other writer is admitted into the gap; otherwise it is released
+ * at once.
+ */
+ if (flush == BTRFS_RESERVE_FLUSH_DATA &&
+ btrfs_test_opt(fs_info, STRIPE_ALLOC)) {
+ const u64 unit = READ_ONCE(fs_info->stripe_margin_unit);
+
+ if (unit)
+ stripe_margin = (u64)count_max_extents(fs_info, len) * unit;
+ }
+
+ ret = btrfs_reserve_data_bytes(data_sinfo_for_inode(inode),
+ len + stripe_margin, flush);
if (ret < 0)
return ret;
+ if (stripe_margin && !stripe_held)
+ btrfs_space_info_free_bytes_may_use(data_sinfo_for_inode(inode),
+ stripe_margin);
+ if (stripe_held)
+ *stripe_held = stripe_margin;
/* Use new btrfs_qgroup_reserve_data to reserve precious data space. */
ret = btrfs_qgroup_reserve_data(inode, reserved, start, len);
if (ret < 0) {
+ if (stripe_held) {
+ btrfs_release_data_margin_probe(inode, stripe_margin);
+ *stripe_held = 0;
+ }
btrfs_free_reserved_data_space_noquota(inode, len);
extent_changeset_free(*reserved);
*reserved = NULL;
return ret;
}
+
+/* Release the probe margin btrfs_check_data_free_space() handed back in @stripe_held. */
+void btrfs_release_data_margin_probe(struct btrfs_inode *inode, u64 held)
+{
+ if (held)
+ btrfs_space_info_free_bytes_may_use(data_sinfo_for_inode(inode), held);
+}
/*
* Called if we need to clear a data reservation for this inode
* Normally in a error case.
int btrfs_delalloc_reserve_space(struct btrfs_inode *inode,
struct extent_changeset **reserved, u64 start, u64 len)
{
+ u64 held = 0;
int ret;
- ret = btrfs_check_data_free_space(inode, reserved, start, len, false);
+ ret = btrfs_check_data_free_space(inode, reserved, start, len, false, &held);
if (ret < 0)
return ret;
ret = btrfs_delalloc_reserve_metadata(inode, len, len, false);
+ btrfs_release_data_margin_probe(inode, held);
if (ret < 0) {
btrfs_free_reserved_data_space(inode, *reserved, start, len);
extent_changeset_free(*reserved);
int btrfs_alloc_data_chunk_ondemand(const struct btrfs_inode *inode, u64 bytes);
int btrfs_check_data_free_space(struct btrfs_inode *inode,
struct extent_changeset **reserved, u64 start, u64 len,
- bool noflush);
+ bool noflush, u64 *stripe_held);
+void btrfs_release_data_margin_probe(struct btrfs_inode *inode, u64 held);
void btrfs_free_reserved_data_space(struct btrfs_inode *inode,
struct extent_changeset *reserved, u64 start, u64 len);
void btrfs_delalloc_release_space(struct btrfs_inode *inode,
struct extent_changeset *data_reserved;
struct btrfs_ordered_extent *ordered;
bool data_space_reserved;
+ /* stripe_alloc probe margin held in bytes_may_use until the metadata reservation */
+ u64 stripe_held;
bool nocow_done;
bool updated_isize;
};
if (write && !(flags & IOMAP_NOWAIT)) {
ret = btrfs_check_data_free_space(BTRFS_I(inode),
&dio_data->data_reserved,
- start, data_alloc_len, false);
+ start, data_alloc_len, false,
+ &dio_data->stripe_held);
if (!ret)
dio_data->data_space_reserved = true;
else if (!(BTRFS_I(inode)->flags &
if (write) {
ret = btrfs_get_blocks_direct_write(&em, inode, dio_data,
start, &len, flags);
+ btrfs_release_data_margin_probe(BTRFS_I(inode), dio_data->stripe_held);
+ dio_data->stripe_held = 0;
if (ret < 0)
goto unlock_err;
/* Recalc len in case the new em is smaller than requested */
btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend,
EXTENT_LOCKED | EXTENT_DIO_LOCKED, &cached_state);
err:
+ btrfs_release_data_margin_probe(BTRFS_I(inode), dio_data->stripe_held);
+ dio_data->stripe_held = 0;
if (dio_data->data_space_reserved) {
btrfs_free_reserved_data_space(BTRFS_I(inode),
dio_data->data_reserved,
(bg->flags & BTRFS_BLOCK_GROUP_RAID56_MASK);
}
+/*
+ * The length to ask this group's stripe allocator for. A piece of a split
+ * allocation (ffe_ctl->split_bytes) is trimmed to whole stripes of this
+ * group: an allocation claims whole stripes, and the claimed stripes' tail
+ * past it is stranded when its run closes. The reservation margin pays
+ * for one such tail per delalloc extent, the final piece's, so every
+ * other piece must leave none. The piece size cannot be chosen before
+ * the group is: groups narrow at the fill edge, where a 448K multiple
+ * claims a third 320K stripe and strands 192K, once per piece. A piece
+ * shorter than one stripe of the group (or whose whole stripes fall short
+ * of the caller's minimum) is left alone.
+ */
+static u64 stripe_split_bytes(const struct btrfs_block_group *block_group,
+ const struct find_free_extent_ctl *ffe_ctl)
+{
+ const u64 fsl = block_group->full_stripe_len;
+ const u64 bytes = ffe_ctl->split_bytes;
+ u64 whole;
+
+ if (!bytes)
+ return ffe_ctl->num_bytes;
+ /* Not a power of two: divide. */
+ whole = div64_u64(bytes, fsl) * fsl;
+ if (!whole || whole < ffe_ctl->min_alloc_size)
+ return bytes;
+ return whole;
+}
+
+/* The trimmed piece was allocated: it is what find_free_extent() returns. */
+static void stripe_split_taken(struct find_free_extent_ctl *ffe_ctl,
+ u64 num_bytes)
+{
+ if (!ffe_ctl->split_bytes)
+ return;
+ ffe_ctl->num_bytes = num_bytes;
+ ffe_ctl->ram_bytes = num_bytes;
+}
+
static int do_allocation_stripe(struct btrfs_block_group *block_group,
struct find_free_extent_ctl *ffe_ctl,
struct btrfs_block_group **bg_ret)
struct btrfs_fs_info *fs_info = block_group->fs_info;
enum btrfs_stripe_run_class class = BTRFS_STRIPE_RUN_COW;
u64 available = 0;
+ u64 num_bytes;
u64 offset;
bool skip = false;
int ret;
* the shared runs: the data is then placed exactly as before 3b.1
* and the log-commit settling keeps it safe, just less cheaply.
*/
+ num_bytes = stripe_split_bytes(block_group, ffe_ctl);
if (ffe_ctl->for_log_inode) {
struct btrfs_inode *log_inode = ffe_ctl->for_log_inode;
if (ret == -ENOMEM)
return ret;
if (!ret) {
+ stripe_split_taken(ffe_ctl, num_bytes);
WRITE_ONCE(log_inode->log_run_hint, offset);
ffe_ctl->found_offset = offset;
ffe_ctl->search_start = offset;
ffe_ctl->max_extent_size = available;
}
- ret = btrfs_alloc_from_open_stripe(block_group, ffe_ctl->num_bytes,
+ ret = btrfs_alloc_from_open_stripe(block_group, num_bytes,
class, &offset, &available);
if (ret == -ENOMEM)
return ret;
btrfs_clear_data_reloc_bg(block_group);
return 1;
}
+ stripe_split_taken(ffe_ctl, num_bytes);
ffe_ctl->found_offset = offset;
ffe_ctl->search_start = offset;
return 0;
struct btrfs_fs_info *fs_info = root->fs_info;
struct find_free_extent_ctl ffe_ctl = {};
bool final_tried = num_bytes == min_alloc_size;
+ u64 split_bytes = 0;
u64 flags;
int ret;
bool for_treelog = (btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID);
ffe_ctl.for_treelog = for_treelog;
ffe_ctl.for_data_reloc = for_data_reloc;
ffe_ctl.for_log_inode = for_log_inode;
+ ffe_ctl.split_bytes = split_bytes;
ret = find_free_extent(root, ins, &ffe_ctl);
if (!ret && !is_data) {
ram_bytes = num_bytes;
if (num_bytes == min_alloc_size)
final_tried = true;
+ /*
+ * stripe_alloc: when a data allocation must be split
+ * across free space fragments, keep every non-final
+ * piece a whole-stripe multiple so at most one piece
+ * per delalloc extent can strand a sub-stripe tail.
+ * The reservation margin pre-paid exactly one tail;
+ * splitting must not mint more. Whole stripes of the
+ * group the piece lands in: do_allocation_stripe()
+ * trims it there (stripe_split_bytes()).
+ */
+ if (is_data && btrfs_test_opt(fs_info, STRIPE_ALLOC))
+ split_bytes = num_bytes;
goto again;
} else if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
struct btrfs_space_info *sinfo;
if (sinfo)
btrfs_dump_space_info(sinfo, num_bytes, 1);
}
+ /*
+ * The pessimistic reservation margin should make a data
+ * writeback allocation failure impossible: admission held
+ * back a full stripe per delalloc extent. If it happens
+ * anyway the pages are dropped, so say so loudly.
+ */
+ if (is_data && delalloc && btrfs_test_opt(fs_info, STRIPE_ALLOC))
+ btrfs_warn_rl(fs_info,
+ "stripe_alloc: data writeback allocation of %llu bytes returned ENOSPC despite reservation margin; buffered data in this range will be dropped",
+ num_bytes);
}
return ret;
/* Max contiguous hole found */
u64 max_extent_size;
+ /*
+ * stripe_alloc: a data allocation that btrfs_reserve_extent() split
+ * because nothing held it whole. The caller takes whatever length
+ * comes back, so a stripe-exclusive group trims the piece to its own
+ * whole stripes; this is the length asked for, before any trim.
+ */
+ u64 split_bytes;
+
/* Total free space from free space cache, not always contiguous */
u64 total_free_space;
const struct btrfs_fs_info *fs_info = inode->root->fs_info;
const unsigned int block_offset = (start & (fs_info->sectorsize - 1));
size_t reserve_bytes;
+ u64 held = 0;
int ret;
- ret = btrfs_check_data_free_space(inode, data_reserved, start, *len, nowait);
+ ret = btrfs_check_data_free_space(inode, data_reserved, start, *len, nowait,
+ &held);
if (ret < 0) {
int can_nocow;
WARN_ON(reserve_bytes == 0);
ret = btrfs_delalloc_reserve_metadata(inode, reserve_bytes,
reserve_bytes, nowait);
+ btrfs_release_data_margin_probe(inode, held);
if (ret) {
if (!*only_release_metadata)
btrfs_free_reserved_data_space(inode, *data_reserved,
struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct extent_io_tree *io_tree = &inode->io_tree;
struct btrfs_ordered_extent *ordered;
+ u64 held = 0;
struct extent_state *cached_state = NULL;
struct extent_changeset *data_reserved = NULL;
unsigned long zero_start;
* being processed by btrfs_page_mkwrite() function.
*/
ret = btrfs_check_data_free_space(inode, &data_reserved, page_start,
- reserved_space, false);
+ reserved_space, false, &held);
if (ret < 0) {
size_t write_bytes = reserved_space;
}
ret = btrfs_delalloc_reserve_metadata(inode, reserved_space,
reserved_space, false);
+ btrfs_release_data_margin_probe(inode, held);
if (ret < 0) {
if (!only_release_metadata)
btrfs_free_reserved_data_space(inode, data_reserved,
static void stripe_unusable_mark_dirty(struct btrfs_block_group *bg);
+/*
+ * Count the sub-stripe head and tail of a returning free range as trapped
+ * at once: reservations read bytes_stripe_unusable mid-transaction, and the
+ * commit rescan only corrects it later. Whole stripes inside the range are
+ * claimable and not counted. The head and tail count even if neighbouring
+ * free space completes their stripes; that over-count (an early ENOSPC
+ * until the rescan) is the safe direction.
+ */
+static void stripe_unusable_account_add(struct btrfs_block_group *bg,
+ u64 bytenr, u64 size)
+{
+ struct btrfs_space_info *sinfo = bg->space_info;
+ const u64 fsl = bg->full_stripe_len;
+ u64 rel_start, rel_end, aligned_start, aligned_end, frag;
+
+ if (!READ_ONCE(bg->stripe_unusable_ready) || !sinfo)
+ return;
+ if (!btrfs_is_stripe_alloc_bg(bg))
+ return;
+
+ rel_start = bytenr - bg->start;
+ rel_end = rel_start + size;
+ aligned_start = div64_u64(rel_start + fsl - 1, fsl) * fsl;
+ aligned_end = div64_u64(rel_end, fsl) * fsl;
+
+ if (aligned_end > aligned_start)
+ frag = size - (aligned_end - aligned_start);
+ else
+ frag = size;
+ if (!frag)
+ return;
+
+ spin_lock(&sinfo->lock);
+ sinfo->bytes_stripe_unusable += frag;
+ spin_unlock(&sinfo->lock);
+}
+
int btrfs_add_free_space(struct btrfs_block_group *block_group,
u64 bytenr, u64 size)
{
trim_state = BTRFS_TRIM_STATE_TRIMMED;
stripe_unusable_mark_dirty(block_group);
+ stripe_unusable_account_add(block_group, bytenr, size);
return __btrfs_add_free_space(block_group, bytenr, size, trim_state);
}
*/
u64 max_extent_size;
+ /*
+ * Widest full-stripe width among raid56 DATA block groups; the unit
+ * of the stripe_alloc pessimistic reservation margin (one unit per
+ * outstanding delalloc extent). Zero when the filesystem has no
+ * raid56 data. Grows monotonically as block groups appear.
+ */
+ u64 stripe_margin_unit;
+
/* Block groups and devices containing active swapfiles. */
spinlock_t swapfile_pins_lock;
struct rb_root swapfile_pins;
struct extent_changeset *data_reserved = NULL;
bool only_release_metadata = false;
u32 blocksize = fs_info->sectorsize;
+ u64 held = 0;
pgoff_t index = (offset >> PAGE_SHIFT);
struct folio *folio;
gfp_t mask = btrfs_alloc_write_mask(mapping);
block_end = block_start + blocksize - 1;
ret = btrfs_check_data_free_space(inode, &data_reserved, block_start,
- blocksize, false);
+ blocksize, false, &held);
if (ret < 0) {
size_t write_bytes = blocksize;
}
}
ret = btrfs_delalloc_reserve_metadata(inode, blocksize, blocksize, false);
+ btrfs_release_data_margin_probe(inode, held);
if (ret < 0) {
if (!only_release_metadata)
btrfs_free_reserved_data_space(inode, data_reserved,
#include "extent-tree.h"
#include "zoned.h"
#include "delayed-inode.h"
+#include "btrfs_inode.h"
/*
* HOW DOES SPACE RESERVATION WORK
factor = btrfs_bg_type_to_factor(block_group->flags);
+ /*
+ * The widest raid56 data full stripe is the unit of the stripe_alloc
+ * reservation margin. Pure geometry: whether a margin is charged is
+ * decided per reservation.
+ */
+ if ((block_group->flags & BTRFS_BLOCK_GROUP_DATA) &&
+ (block_group->flags & BTRFS_BLOCK_GROUP_RAID56_MASK) &&
+ block_group->full_stripe_len >
+ READ_ONCE(block_group->fs_info->stripe_margin_unit))
+ WRITE_ONCE(block_group->fs_info->stripe_margin_unit,
+ block_group->full_stripe_len);
+
spin_lock(&space_info->lock);
if (!(block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED) ||
up_write(&space_info->groups_sem);
}
+/*
+ * stripe_alloc pessimistic reservation margin.
+ *
+ * Between admission at write() and allocation at writeback, each commit
+ * closes open stripe runs and traps their sub-stripe tails, turning bytes a
+ * data reservation counted on into bytes_stripe_unusable, and delalloc
+ * writeback cannot fail cleanly. So pre-pay the worst case: one full stripe
+ * per outstanding delalloc extent, released when the extent is allocated or
+ * its delalloc range dropped.
+ *
+ * The charge rides btrfs_mod_outstanding_extents(), so it stays exact across
+ * delalloc merges and splits. Charges require stripe_alloc and releases do
+ * not (they clamp at zero), so disabling the policy with dirty delalloc
+ * drains the counter instead of stranding it.
+ */
+void btrfs_stripe_margin_mod(struct btrfs_inode *inode, int mod)
+{
+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
+ struct btrfs_space_info *sinfo = fs_info->data_sinfo;
+ const u64 unit = READ_ONCE(fs_info->stripe_margin_unit);
+ u64 bytes;
+
+ if (!unit || !sinfo || mod == 0)
+ return;
+ if (btrfs_test_opt(fs_info, STRIPE_ALLOC)) {
+ /* charges and releases both live */
+ } else if (mod > 0 || READ_ONCE(sinfo->bytes_stripe_margin) == 0) {
+ /*
+ * stripe_alloc off: never charge, and skip the lock once the
+ * counter has drained (with no charges it only falls, so
+ * reading zero is stable).
+ */
+ return;
+ }
+
+ bytes = (u64)abs(mod) * unit;
+ spin_lock(&sinfo->lock);
+ if (mod > 0) {
+ sinfo->bytes_stripe_margin += bytes;
+ } else {
+ sinfo->bytes_stripe_margin -= min(bytes, sinfo->bytes_stripe_margin);
+ btrfs_try_granting_tickets(sinfo);
+ }
+ spin_unlock(&sinfo->lock);
+}
+
+
struct btrfs_space_info *btrfs_find_space_info(const struct btrfs_fs_info *info,
u64 flags)
{
(s64)(info->total_bytes - btrfs_space_info_used(info, true)),
info->full ? "" : "not ");
btrfs_info(fs_info,
-"space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu zone_unusable=%llu",
+"space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu zone_unusable=%llu stripe_unusable=%llu stripe_open=%llu stripe_margin=%llu",
info->total_bytes, info->bytes_used, info->bytes_pinned,
info->bytes_reserved, info->bytes_may_use,
- info->bytes_readonly, info->bytes_zone_unusable);
+ info->bytes_readonly, info->bytes_zone_unusable,
+ info->bytes_stripe_unusable, info->bytes_stripe_open,
+ info->bytes_stripe_margin);
}
void btrfs_dump_space_info(struct btrfs_space_info *info, u64 bytes,
reservations must not count on it
either. Synced from the block groups'
stripe_open_remainder. */
+ u64 bytes_stripe_margin; /* stripe_alloc pessimistic reservation
+ margin: one full stripe width per
+ outstanding delalloc extent. Pre-pays
+ the worst-case sub-stripe tail each
+ future extent can strand when its run
+ closes, so commits between write() and
+ writeback can never trap space out
+ from under an admitted reservation.
+ Charged and released in lockstep with
+ the inodes' outstanding_extents. */
u64 max_extent_size; /* This will hold the maximum extent size of
the space info if we had an ENOSPC in the
s_info->bytes_pinned + s_info->bytes_readonly +
s_info->bytes_zone_unusable +
s_info->bytes_stripe_unusable + s_info->bytes_stripe_open +
+ s_info->bytes_stripe_margin +
(may_use_included ? s_info->bytes_may_use : 0);
}
+struct btrfs_inode;
+
int btrfs_init_space_info(struct btrfs_fs_info *fs_info);
+void btrfs_stripe_margin_mod(struct btrfs_inode *inode, int mod);
void btrfs_add_bg_to_space_info(struct btrfs_fs_info *info,
struct btrfs_block_group *block_group);
void btrfs_update_space_info_chunk_size(struct btrfs_space_info *space_info,
SPACE_INFO_ATTR(bytes_readonly);
SPACE_INFO_ATTR(bytes_zone_unusable);
SPACE_INFO_ATTR(bytes_stripe_unusable);
+SPACE_INFO_ATTR(bytes_stripe_open);
+SPACE_INFO_ATTR(bytes_stripe_margin);
SPACE_INFO_ATTR(disk_used);
SPACE_INFO_ATTR(disk_total);
SPACE_INFO_ATTR(reclaim_count);
BTRFS_ATTR_PTR(space_info, bytes_readonly),
BTRFS_ATTR_PTR(space_info, bytes_zone_unusable),
BTRFS_ATTR_PTR(space_info, bytes_stripe_unusable),
+ BTRFS_ATTR_PTR(space_info, bytes_stripe_open),
+ BTRFS_ATTR_PTR(space_info, bytes_stripe_margin),
BTRFS_ATTR_PTR(space_info, disk_used),
BTRFS_ATTR_PTR(space_info, disk_total),
BTRFS_ATTR_PTR(space_info, bg_reclaim_threshold),