From 2ed1a60e0815c52abfc8bf558716403b57bfda3b Mon Sep 17 00:00:00 2001 From: Zygo Blaxell Date: Wed, 29 Jul 2026 21:43:12 -0400 Subject: [PATCH] btrfs: stripe_alloc: private per-inode stripe runs for log-active inodes Closing the open stripe run covering every logged extent at each log commit made completed fsyncs crash-safe, but at a placement cost: an fsync-heavy inode shares the open band runs with every other writer, so each of its log commits closes a shared run, fragmenting concurrent write streams and trapping the shared run's tail. Give log-active inodes their own runs instead. The first fsync of an inode sets a sticky runtime flag (at btrfs_sync_file entry, before the fsync flushes its own delalloc, so even that first fsync's allocations are steered). Datacow allocations for a flagged inode come from a private LOG-class run: claimed like any run but never placed in the shared band slots, owned by the inode, and found by owner lookup under the block group's stripe_run_lock. A per-inode hint seeds the allocator's search with the run's location; the hint is advisory (a stale hint costs a lookup miss, never a wrong run, since the owner match is authoritative). When no fully-free stripes remain for a private run the allocation falls back to the shared runs, restoring the previous placement with unchanged safety. One inode's log commit now settles only its own stripes: other writers' runs stay open and their streams stay contiguous, and the only trapped tails are the fsyncing inode's own. Private runs also give each log-active inode exclusive stripes, which a later change uses to copy live tails forward and reclaim them without touching foreign extents. The commit-time retirement closed runs by walking the band slots, which private runs never occupy; walk the block group's run list instead so every open run, slotted or private, is closed and drained under invariant I2 (no open run survives a transaction commit). Assisted-by: Claude:claude-fable-5 --- fs/btrfs/block-group.c | 154 ++++++++++++++++++++++++++++++++++++----- fs/btrfs/block-group.h | 10 +++ fs/btrfs/btrfs_inode.h | 21 ++++++ fs/btrfs/direct-io.c | 2 +- fs/btrfs/extent-tree.c | 49 ++++++++++++- fs/btrfs/extent-tree.h | 9 ++- fs/btrfs/file.c | 11 +++ fs/btrfs/inode.c | 9 +-- 8 files changed, 239 insertions(+), 26 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 1a208a389491b..b699889742775 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -467,6 +467,7 @@ struct btrfs_open_stripe_run { u64 offset; /* next unallocated byte */ u64 inflight_bytes; /* reserved bytes with data IO pending */ u64 open_seq; /* fs_info->stripe_retire_seq at open */ + u64 owner; /* LOG class: btrfs_ino of the owner */ enum btrfs_stripe_run_class class; bool open; /* accepting allocations */ }; @@ -541,6 +542,9 @@ static u64 stripe_run_place(struct btrfs_block_group *bg, int band = stripe_run_band(bg, run->end - run->offset); struct btrfs_open_stripe_run *occ = bg->open_stripe[class][band]; + /* Per-inode LOG runs are private and never occupy a band slot. */ + ASSERT(class != BTRFS_STRIPE_RUN_LOG); + if (occ) { struct btrfs_open_stripe_run *keep, *shut; @@ -715,6 +719,7 @@ int btrfs_alloc_from_open_stripe(struct btrfs_block_group *bg, u64 num_bytes, } new_run->bg = bg; new_run->class = class; + new_run->owner = 0; new_run->start = start; new_run->end = start + len; new_run->offset = start + num_bytes; @@ -742,6 +747,116 @@ out: return ret; } +/* + * Allocate num_bytes for a log-active inode from its private LOG-class run + * in this block group, claiming a fresh run when the inode has none here or + * the current one does not fit. Private runs are never placed in the shared + * band slots and are found by owner lookup, so no other allocation can join + * an inode's run and one inode's log commit settles only its own stripes. + * + * A replaced run (too small for this allocation) is closed immediately: the + * inode moves on to the fresh run, so keeping the remainder open would only + * strand it until the commit-time retirement. Two racing allocations for + * the same inode can each claim a fresh run; the owner walk then serves one + * of them and the other drains and retires at the next commit -- an + * occasional trapped tail, not a correctness problem. + * + * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in + * this block group can satisfy the allocation, or -ENOMEM. + */ +int btrfs_alloc_from_inode_stripe_run(struct btrfs_block_group *bg, u64 ino, + u64 num_bytes, u64 *ret_offset, + u64 *available) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + const u64 fsl = bg->full_stripe_len; + struct btrfs_open_stripe_run *new_run; + struct btrfs_open_stripe_run *run; + unsigned long flags; + u64 tail_start = 0; + u64 tail_len = 0; + u64 start; + u64 len; + int ret; + + ASSERT(num_bytes); + *available = 0; + + spin_lock_irqsave(&bg->stripe_run_lock, flags); + list_for_each_entry(run, &bg->open_stripe_runs, list) { + if (run->class != BTRFS_STRIPE_RUN_LOG || run->owner != ino || + !run->open) + continue; + if (num_bytes <= run->end - run->offset) { + *ret_offset = run->offset; + run->offset += num_bytes; + run->inflight_bytes += num_bytes; + if (run->offset == run->end) + run->open = false; /* full; drains, is freed */ + spin_unlock_irqrestore(&bg->stripe_run_lock, flags); + return 0; + } + tail_len = close_open_stripe_run(bg, run, &tail_start); + break; + } + spin_unlock_irqrestore(&bg->stripe_run_lock, flags); + if (tail_len) + btrfs_add_free_space(bg, tail_start, tail_len); + + new_run = kmalloc(sizeof(*new_run), GFP_NOFS); + if (!new_run) + return -ENOMEM; + + ret = btrfs_claim_free_stripe_run(bg, + div64_u64(num_bytes + fsl - 1, fsl) * fsl, + &start, &len); + if (ret) + goto out; + if (len < num_bytes) { + /* Only a shorter run is free; see btrfs_alloc_from_open_stripe(). */ + btrfs_add_free_space(bg, start, len); + *available = len; + ret = -ENOSPC; + goto out; + } + + /* See the installation comment in btrfs_alloc_from_open_stripe(). */ + spin_lock(&fs_info->open_stripe_lock); + if (list_empty(&bg->open_stripe_bg_list)) { + btrfs_get_block_group(bg); + list_add_tail(&bg->open_stripe_bg_list, + &fs_info->open_stripe_bgs); + } + spin_lock_irqsave(&bg->stripe_run_lock, flags); + if (bg->ro) { + spin_unlock_irqrestore(&bg->stripe_run_lock, flags); + spin_unlock(&fs_info->open_stripe_lock); + btrfs_add_free_space(bg, start, len); + *available = 0; + ret = -ENOSPC; + goto out; + } + new_run->bg = bg; + new_run->class = BTRFS_STRIPE_RUN_LOG; + new_run->owner = ino; + new_run->start = start; + new_run->end = start + len; + new_run->offset = start + num_bytes; + new_run->inflight_bytes = num_bytes; + new_run->open_seq = fs_info->stripe_retire_seq; + new_run->open = (new_run->offset != new_run->end); + list_add_tail(&new_run->list, &bg->open_stripe_runs); + spin_unlock_irqrestore(&bg->stripe_run_lock, flags); + spin_unlock(&fs_info->open_stripe_lock); + + *ret_offset = start; + new_run = NULL; + ret = 0; +out: + kfree(new_run); + return ret; +} + /* * A live stripe run's address range must not be re-claimed while the run * object still exists: an ENOSPC discard storm can return every byte of a @@ -1107,30 +1222,35 @@ void btrfs_open_stripe_write_done_run(struct btrfs_open_stripe_run *run, static void close_block_group_stripe_runs(struct btrfs_block_group *bg, u64 seq) { - u64 tail_start[BTRFS_STRIPE_RUN_NR_CLASSES][BTRFS_STRIPE_RUN_NR_BANDS]; - u64 tail_len[BTRFS_STRIPE_RUN_NR_CLASSES][BTRFS_STRIPE_RUN_NR_BANDS]; struct btrfs_open_stripe_run *run; unsigned long flags; int class, band; +restart: spin_lock_irqsave(&bg->stripe_run_lock, flags); - for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++) { - for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) { - tail_len[class][band] = 0; - run = bg->open_stripe[class][band]; - if (run && run->open_seq < seq) { - bg->open_stripe[class][band] = NULL; - tail_len[class][band] = close_open_stripe_run( - bg, run, &tail_start[class][band]); - } - } + list_for_each_entry(run, &bg->open_stripe_runs, list) { + u64 tail_start; + u64 tail_len; + + if (!run->open || run->open_seq >= seq) + continue; + for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++) + for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) + if (bg->open_stripe[class][band] == run) + bg->open_stripe[class][band] = NULL; + tail_len = close_open_stripe_run(bg, run, &tail_start); + /* + * Closing may have freed the run, and returning its tail to + * the free space cache needs the lock dropped, so rescan. + * Each pass closes one run and open runs are bounded (band + * slots plus the private runs of inodes logged this window). + */ + spin_unlock_irqrestore(&bg->stripe_run_lock, flags); + if (tail_len) + btrfs_add_free_space(bg, tail_start, tail_len); + goto restart; } spin_unlock_irqrestore(&bg->stripe_run_lock, flags); - for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++) - for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) - if (tail_len[class][band]) - btrfs_add_free_space(bg, tail_start[class][band], - tail_len[class][band]); } /* diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index a7e8c800c982b..466abd3f2fa15 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -30,6 +30,13 @@ struct btrfs_trans_handle; enum btrfs_stripe_run_class { BTRFS_STRIPE_RUN_COW, BTRFS_STRIPE_RUN_RELOC, + /* + * Private per-inode runs for log-active (fsync-heavy) inodes. Never + * placed in the shared band slots: each run is owned by one inode and + * found by owner lookup, so one inode's log commit settles only its + * own stripes. + */ + BTRFS_STRIPE_RUN_LOG, BTRFS_STRIPE_RUN_NR_CLASSES, }; @@ -381,6 +388,9 @@ void btrfs_wait_block_group_reservations(struct btrfs_block_group *bg); int btrfs_alloc_from_open_stripe(struct btrfs_block_group *bg, u64 num_bytes, enum btrfs_stripe_run_class class, u64 *ret_offset, u64 *available); +int btrfs_alloc_from_inode_stripe_run(struct btrfs_block_group *bg, u64 ino, + u64 num_bytes, u64 *ret_offset, + u64 *available); void btrfs_open_stripe_write_done(struct btrfs_block_group *bg, u64 start, u64 num_bytes); void btrfs_open_stripe_write_done_run(struct btrfs_open_stripe_run *run, diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index b6b46f951f11a..9797d85d77844 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -86,6 +86,17 @@ enum { BTRFS_INODE_VERITY_IN_PROGRESS, /* Set when this inode is a free space inode. */ BTRFS_INODE_FREE_SPACE_INODE, + /* + * Set the first time this inode is fsynced under stripe-exclusive + * allocation and never cleared while the inode stays cached. A + * log-active inode's datacow allocations are steered into the inode's + * own private LOG-class stripe run, so that its log commits settle + * only its own stripes instead of closing (and trapping the tails of) + * runs shared with other writers. Purely an allocation placement + * hint: correctness comes from the log-commit settling itself, which + * works on runs of any class. + */ + BTRFS_INODE_LOG_ALLOC, /* Set when there are no capabilities in XATTs for the inode. */ BTRFS_INODE_NO_CAP_XATTR, /* @@ -321,6 +332,16 @@ struct btrfs_inode { u64 ref_root_id; }; + /* + * Start of the last extent allocated from this inode's private + * LOG-class stripe run (see BTRFS_INODE_LOG_ALLOC). Advisory only: + * it seeds the allocator's search hint so consecutive allocations + * find the block group holding the inode's run. The run itself is + * found by owner lookup under the block group's stripe_run_lock, so a + * stale hint costs a miss, never a wrong run. + */ + u64 log_run_hint; + /* Backwards incompatible flags, lower half of inode_item::flags */ u32 flags; /* Read-only compatibility flags, upper half of inode_item::flags */ diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index 5f2a99e683353..7e946308227cb 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -187,7 +187,7 @@ static struct extent_map *btrfs_new_extent_direct(struct btrfs_inode *inode, alloc_hint = btrfs_get_extent_allocation_hint(inode, start, len); again: - ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize, + ret = btrfs_reserve_extent(root, inode, len, len, fs_info->sectorsize, 0, alloc_hint, &ins, 1, 1); if (ret == -EAGAIN) { ASSERT(btrfs_is_zoned(fs_info)); diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index c3265fc2771d4..517eeb2b11899 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -4056,6 +4056,31 @@ static int do_allocation_stripe(struct btrfs_block_group *block_group, if (skip) return 1; + /* + * A log-active inode allocates from its own private LOG-class run so + * its log commits settle only its own stripes. If this block group + * has no fully-free stripes left for a private run, fall through to + * the shared runs: the data is then placed exactly as before 3b.1 + * and the log-commit settling keeps it safe, just less cheaply. + */ + if (ffe_ctl->for_log_inode) { + struct btrfs_inode *log_inode = ffe_ctl->for_log_inode; + + ret = btrfs_alloc_from_inode_stripe_run(block_group, + btrfs_ino(log_inode), ffe_ctl->num_bytes, + &offset, &available); + if (ret == -ENOMEM) + return ret; + if (!ret) { + WRITE_ONCE(log_inode->log_run_hint, offset); + ffe_ctl->found_offset = offset; + ffe_ctl->search_start = offset; + return 0; + } + if (available > ffe_ctl->max_extent_size) + ffe_ctl->max_extent_size = available; + } + ret = btrfs_alloc_from_open_stripe(block_group, ffe_ctl->num_bytes, class, &offset, &available); if (ret == -ENOMEM) @@ -4767,8 +4792,8 @@ loop: * case -ENOSPC is returned then @ins->offset will contain the size of the * largest available hole the allocator managed to find. */ -int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes, - u64 num_bytes, u64 min_alloc_size, +int btrfs_reserve_extent(struct btrfs_root *root, struct btrfs_inode *inode, + u64 ram_bytes, u64 num_bytes, u64 min_alloc_size, u64 empty_size, u64 hint_byte, struct btrfs_key *ins, int is_data, int delalloc) { @@ -4779,6 +4804,23 @@ int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes, int ret; bool for_treelog = (btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID); bool for_data_reloc = (btrfs_is_data_reloc_root(root) && is_data); + struct btrfs_inode *for_log_inode = NULL; + + /* + * Steer a log-active inode's datacow data into its private LOG-class + * stripe run, seeded with the run's location as the search hint; see + * BTRFS_INODE_LOG_ALLOC. Purely placement: any allocation that + * cannot be served from a private run falls back to the shared runs. + */ + if (inode && is_data && !for_data_reloc && + btrfs_test_opt(fs_info, STRIPE_ALLOC) && + test_bit(BTRFS_INODE_LOG_ALLOC, &inode->runtime_flags)) { + u64 log_hint = READ_ONCE(inode->log_run_hint); + + for_log_inode = inode; + if (log_hint) + hint_byte = log_hint; + } flags = get_alloc_profile_by_root(root, is_data); again: @@ -4793,6 +4835,7 @@ again: ffe_ctl.hint_byte = hint_byte; ffe_ctl.for_treelog = for_treelog; ffe_ctl.for_data_reloc = for_data_reloc; + ffe_ctl.for_log_inode = for_log_inode; ret = find_free_extent(root, ins, &ffe_ctl); if (!ret && !is_data) { @@ -5277,7 +5320,7 @@ struct extent_buffer *btrfs_alloc_tree_block(struct btrfs_trans_handle *trans, if (IS_ERR(block_rsv)) return ERR_CAST(block_rsv); - ret = btrfs_reserve_extent(root, blocksize, blocksize, blocksize, + ret = btrfs_reserve_extent(root, NULL, blocksize, blocksize, blocksize, empty_size, hint, &ins, 0, 0); if (ret) goto out_unuse; diff --git a/fs/btrfs/extent-tree.h b/fs/btrfs/extent-tree.h index f37a307714b09..0bdbfe2734a72 100644 --- a/fs/btrfs/extent-tree.h +++ b/fs/btrfs/extent-tree.h @@ -49,6 +49,12 @@ struct find_free_extent_ctl { /* Allocation is called for data relocation */ bool for_data_reloc; + /* + * Data allocation for a log-active inode under stripe-exclusive + * allocation: steer it into the inode's private LOG-class stripe run. + */ + struct btrfs_inode *for_log_inode; + /* RAID index, converted from flags */ int index; @@ -136,7 +142,8 @@ int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans, int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans, u64 root_objectid, u64 owner, u64 offset, struct btrfs_key *ins); -int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes, u64 num_bytes, +int btrfs_reserve_extent(struct btrfs_root *root, struct btrfs_inode *inode, + u64 ram_bytes, u64 num_bytes, u64 min_alloc_size, u64 empty_size, u64 hint_byte, struct btrfs_key *ins, int is_data, int delalloc); int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index a15062f976ba3..df14b72320e26 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1545,6 +1545,17 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) trace_btrfs_sync_file(file, datasync); + /* + * Under stripe-exclusive allocation, mark the inode log-active before + * this fsync flushes its delalloc, so that even the first fsync's + * allocations are steered into the inode's own LOG-class stripe run + * (see BTRFS_INODE_LOG_ALLOC). Sticky while the inode stays cached: + * one fsync is taken as a signal that more will follow. + */ + if (btrfs_test_opt(fs_info, STRIPE_ALLOC) && + !test_bit(BTRFS_INODE_LOG_ALLOC, &inode->runtime_flags)) + set_bit(BTRFS_INODE_LOG_ALLOC, &inode->runtime_flags); + btrfs_init_log_ctx(&ctx, inode); /* diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 88a4158fca975..df6fe0ea75508 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -1147,7 +1147,7 @@ static void submit_one_async_extent(struct async_chunk *async_chunk, goto done; } - ret = btrfs_reserve_extent(root, async_extent->ram_size, + ret = btrfs_reserve_extent(root, inode, async_extent->ram_size, async_extent->compressed_size, async_extent->compressed_size, 0, *alloc_hint, &ins, 1, 1); @@ -1366,7 +1366,7 @@ static noinline int cow_file_range(struct btrfs_inode *inode, struct btrfs_ordered_extent *ordered; struct btrfs_file_extent file_extent; - ret = btrfs_reserve_extent(root, num_bytes, num_bytes, + ret = btrfs_reserve_extent(root, inode, num_bytes, num_bytes, min_alloc_size, 0, alloc_hint, &ins, 1, 1); if (ret == -EAGAIN) { @@ -8036,6 +8036,7 @@ struct inode *btrfs_alloc_inode(struct super_block *sb) ei->last_unlink_trans = 0; ei->last_reflink_trans = 0; ei->last_log_commit = 0; + ei->log_run_hint = 0; spin_lock_init(&ei->lock); ei->outstanding_extents = 0; @@ -9218,7 +9219,7 @@ static int __btrfs_prealloc_file_range(struct inode *inode, int mode, * sized chunks. */ cur_bytes = min(cur_bytes, last_alloc); - ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes, + ret = btrfs_reserve_extent(root, NULL, cur_bytes, cur_bytes, min_size, 0, *alloc_hint, &ins, 1, 0); if (ret) break; @@ -10039,7 +10040,7 @@ ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from, } } - ret = btrfs_reserve_extent(root, disk_num_bytes, disk_num_bytes, + ret = btrfs_reserve_extent(root, inode, disk_num_bytes, disk_num_bytes, disk_num_bytes, 0, 0, &ins, 1, 1); if (ret) goto out_delalloc_release; -- 2.53.0