From: Zygo Blaxell Date: Sat, 8 Aug 2026 15:27:36 +0000 (-0400) Subject: btrfs: stripe_alloc: complete parked writes on allocation coverage, not clocks X-Git-Url: http://git.hungrycats.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=374e640f409ff426bef45faddc588d5a42bf37bf;p=linux btrfs: stripe_alloc: complete parked writes on allocation coverage, not clocks A parked partial-stripe write goes down its RMW path when its deadline expires or a retirement flush kicks it, even when every byte it is missing below the run's frontier belongs to an allocation whose data IO is already in flight -- allocation happens at writeback submission, and the commit's retirement drain waits for exactly those arrivals. Timing out such a park buys nothing and costs a stripe read plus a second write of the same stripe: measured on an 8-device raid5 buffered fill, the 100ms deadline turns ~2200 parked stripes per 2GB into read-modify- writes (pad_decline_live), and stretching the clock 20x recovers only 14% -- the clock is the wrong instrument. Replace the clock with a coverage test. A parked rbio is ready when its gathered bios cover everything the covering run has allocated inside its stripe (btrfs_stripe_run_alloc_ceiling); the remainder lies at or past the frontier, so the existing pad turns it into a single full-stripe write with no read phase. Merges check readiness as they land, the park timer holds unready parks instead of expiring them, and run-retirement flushes leave unready parks parked: the retirement has already closed the run (freezing the allocated prefix) and its drain waits on the very arrivals that will complete them. Waiter-driven flushes (fsync's pre-writeback kick, ordered-extent waits) and sync parks keep today's forced behaviour: a blocked waiter's latency beats a saved stripe read. A stuck cap (10x the park deadline) bounds the wait when an arrival can never come: a writeback error abandoned the allocation, or the stripe was already written once by a forced sync park. Such parks are forced down the old path and counted. New sysfs stripe_park_stats counters: unparked_ready (parks completed by the coverage test) and stuck (parks forced at the cap). Assisted-by: Claude:claude-fable-5 --- diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 587490d58d37e..c2b3193e2fb90 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1309,6 +1309,43 @@ bool btrfs_stripe_nocow_writable(struct btrfs_inode *inode, u64 start, * later allocation's write serializes behind the caller's stripe lock * and overwrites the zeros. */ +/* + * How far has the covering run allocated into this stripe? Returns the + * exclusive end of the allocated prefix, clamped to the stripe, or U64_MAX + * when no run covers the stripe (nothing left to wait for; the caller + * proceeds as if the stripe were fully allocated and fully arrived). + * A partial write below the returned ceiling is waiting on neighbours whose + * data IO is already in flight -- allocation happens at writeback submission + * and the run's inflight accounting makes the commit wait for it -- so a + * parked rbio covering [stripe_start, ceiling) can complete (padding the + * dead tail), and one that does not should keep waiting for merges. + */ +u64 btrfs_stripe_run_alloc_ceiling(struct btrfs_fs_info *fs_info, + u64 stripe_start, u64 stripe_len) +{ + struct btrfs_block_group *bg; + struct btrfs_open_stripe_run *run; + unsigned long flags; + u64 ceiling = U64_MAX; + + if (list_empty_careful(&fs_info->open_stripe_bgs)) + return U64_MAX; + bg = btrfs_lookup_block_group(fs_info, stripe_start); + if (!bg) + return U64_MAX; + spin_lock_irqsave(&bg->stripe_run_lock, flags); + list_for_each_entry(run, &bg->open_stripe_runs, list) { + if (stripe_start < run->start || stripe_start >= run->end) + continue; + ceiling = clamp(run->offset, stripe_start, + stripe_start + stripe_len); + break; + } + spin_unlock_irqrestore(&bg->stripe_run_lock, flags); + btrfs_put_block_group(bg); + return ceiling; +} + bool btrfs_stripe_run_pad_start(struct btrfs_fs_info *fs_info, u64 stripe_start, u64 stripe_len, u64 *pad_from, @@ -1651,7 +1688,7 @@ void btrfs_log_settle_stripes(struct btrfs_inode *inode, u64 file_start, stripe_open_remainder_sync(bg); } if (flush_len) { - btrfs_flush_parked_rbios(fs_info, flush_start, flush_len); + btrfs_flush_parked_rbios(fs_info, flush_start, flush_len, false); wait_var_event(&bg->open_stripe_runs, stripe_log_range_settled(bg, bytenr)); } @@ -2010,7 +2047,7 @@ void btrfs_retire_block_group_stripes(struct btrfs_block_group *bg) { close_block_group_stripe_runs(bg, U64_MAX); /* Parked partial-stripe rbios hold the very bios the wait drains. */ - btrfs_flush_parked_rbios(bg->fs_info, bg->start, bg->length); + btrfs_flush_parked_rbios(bg->fs_info, bg->start, bg->length, false); wait_var_event(&bg->open_stripe_runs, bg_open_stripes_settled(bg, U64_MAX)); } @@ -2052,7 +2089,7 @@ void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info, * A racing park that saw its run still open lands after this flush * and is bounded by the park timer, not by us. */ - btrfs_flush_parked_rbios(fs_info, 0, U64_MAX); + btrfs_flush_parked_rbios(fs_info, 0, U64_MAX, false); while (!list_empty(&retire_list)) { bg = list_first_entry(&retire_list, struct btrfs_block_group, diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 1d75703ddb0a7..46f4ba48bcff8 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -468,6 +468,8 @@ struct btrfs_stripe_pad_info { u8 reason; }; +u64 btrfs_stripe_run_alloc_ceiling(struct btrfs_fs_info *fs_info, + u64 stripe_start, u64 stripe_len); bool btrfs_stripe_run_pad_start(struct btrfs_fs_info *fs_info, u64 stripe_start, u64 stripe_len, u64 *pad_from, diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 94eedf224daa8..1fa03904035e6 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1678,7 +1678,8 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) if (tmp_oe->disk_num_bytes) btrfs_flush_parked_rbios(fs_info, tmp_oe->disk_bytenr, - tmp_oe->disk_num_bytes); + tmp_oe->disk_num_bytes, + true); } ret = filemap_fdatawait_range(inode->vfs_inode.i_mapping, start, end); diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h index 919851032b046..9ea03de9c18da 100644 --- a/fs/btrfs/fs.h +++ b/fs/btrfs/fs.h @@ -824,8 +824,20 @@ struct btrfs_fs_info { atomic64_t parked; atomic64_t merged_bytes; atomic64_t filled; + /* + * Parks released because their write covers everything the + * run has allocated in the stripe -- the rest is dead space + * the pad fills; nothing was worth waiting for. + */ + atomic64_t unparked_ready; atomic64_t expired; atomic64_t kicked; + /* + * Parks forced down past the stuck cap while still missing + * allocated neighbours: an arrival never came (writeback + * error, or a stripe written twice by a forced sync park). + */ + atomic64_t stuck; atomic64_t rmw_reads; atomic64_t padded; /* diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index e7aeee5c7172d..8e06a4266b3a3 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -937,7 +937,7 @@ void btrfs_start_ordered_extent_nowriteback(struct btrfs_ordered_extent *entry, !test_bit(BTRFS_ORDERED_COMPLETE, &entry->flags)) { btrfs_flush_parked_rbios(inode->root->fs_info, entry->disk_bytenr, - entry->disk_num_bytes); + entry->disk_num_bytes, true); if (wait_event_timeout(entry->wait, test_bit(BTRFS_ORDERED_COMPLETE, &entry->flags), diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 2a439066a2dd5..2fbafc7299fef 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -66,6 +66,13 @@ /* Set at park time and never cleared: this rbio once parked (stats). */ #define RBIO_WAS_PARKED_BIT 7 +/* + * Set when a parked rbio carries (or merged) a REQ_SYNC bio: a waiter + * is blocked on it, so deadlines stay authoritative and neither the + * timer nor a flush may hold it back to wait for neighbours. + */ +#define RBIO_SYNC_PARK_BIT 8 + #define RBIO_CACHE_SIZE 1024 /* @@ -780,6 +787,29 @@ static bool rbio_has_sync_bio(struct btrfs_raid_bio *rbio) return false; } +/* + * A parked rbio is ready to go down when its gathered bios cover every + * byte the covering run has allocated in its stripe: the remainder lies + * at or past the frontier and the pad turns it into a full-stripe write. + * Anything less is waiting on an allocated neighbour whose data IO is + * already in flight, so waiting is bounded and reading (RMW) is waste. + * bio_list_bytes is read unlocked: merges only grow it, and a stale low + * value just delays readiness by one scan tick. + */ +static bool parked_rbio_ready(struct btrfs_raid_bio *rbio) +{ + struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; + const u64 stripe_start = rbio->bioc->full_stripe_logical; + const u64 stripe_len = (u64)rbio->nr_data * BTRFS_STRIPE_LEN; + u64 ceiling; + + ceiling = btrfs_stripe_run_alloc_ceiling(fs_info, stripe_start, + stripe_len); + if (ceiling == U64_MAX) + return true; + return READ_ONCE(rbio->bio_list_bytes) >= ceiling - stripe_start; +} + /* * Park a partial write rbio that owns its stripe lock, if its stripe is * covered by an open stripe run. Returns true if the rbio was parked (or @@ -835,7 +865,11 @@ static bool rbio_try_park(struct btrfs_raid_bio *rbio) spin_lock(&table->parked_lock); set_bit(RBIO_PARKED_BIT, &rbio->flags); set_bit(RBIO_WAS_PARKED_BIT, &rbio->flags); + if (timeout_ms == sync_timeout_ms) + set_bit(RBIO_SYNC_PARK_BIT, &rbio->flags); rbio->park_deadline = jiffies + msecs_to_jiffies(timeout_ms); + rbio->park_stuck_deadline = jiffies + + msecs_to_jiffies(timeout_ms * 10); list_add_tail(&rbio->parked_node, &table->parked); spin_unlock(&table->parked_lock); atomic64_inc(&fs_info->stripe_park_stats.parked); @@ -864,7 +898,8 @@ static bool rbio_try_park(struct btrfs_raid_bio *rbio) * write. Callers may hold the stripe hash and bio_list locks; parked_lock * nests inside both. */ -static void unpark_full_rbio(struct btrfs_raid_bio *rbio) +static void unpark_ready_rbio(struct btrfs_raid_bio *rbio, + atomic64_t *counter) { struct btrfs_stripe_hash_table *table = rbio->bioc->fs_info->stripe_hash_table; @@ -876,7 +911,7 @@ static void unpark_full_rbio(struct btrfs_raid_bio *rbio) } rbio_unpark_locked(rbio); spin_unlock(&table->parked_lock); - atomic64_inc(&rbio->bioc->fs_info->stripe_park_stats.filled); + atomic64_inc(counter); start_async_work(rbio, rmw_rbio_work_locked); } @@ -886,7 +921,7 @@ static void unpark_full_rbio(struct btrfs_raid_bio *rbio) * ordered-extent waiters, and the park timer. */ void btrfs_flush_parked_rbios(struct btrfs_fs_info *fs_info, u64 start, - u64 num_bytes) + u64 num_bytes, bool force) { struct btrfs_stripe_hash_table *table = fs_info->stripe_hash_table; struct btrfs_raid_bio *rbio; @@ -903,6 +938,17 @@ void btrfs_flush_parked_rbios(struct btrfs_fs_info *fs_info, u64 start, if (rbio_start + rbio_len <= start || (num_bytes != U64_MAX && rbio_start >= start + num_bytes)) continue; + /* + * A run-retirement flush precedes the commit's IO drain, + * which waits for the very arrivals this rbio is missing: + * leave it parked and the arrivals will complete it (the + * run is closed, so its allocated prefix is frozen). Only + * waiter-driven flushes (fsync, ordered-extent waits) force + * an incomplete stripe down, and sync parks always go. + */ + if (!force && !test_bit(RBIO_SYNC_PARK_BIT, &rbio->flags) && + !parked_rbio_ready(rbio)) + continue; rbio_unpark_locked(rbio); list_add_tail(&rbio->parked_node, &flush); } @@ -933,14 +979,35 @@ static void parked_rbios_timeout_work(struct work_struct *work) rearm = true; continue; } + /* + * Past the deadline but still missing allocated + * neighbours: their data IO is in flight, so expiring + * now would read (RMW) what a short wait merges for + * free. Hold the park up to the stuck cap -- the cap + * only fires when an arrival never comes (writeback + * error, or a stripe already written once by a forced + * sync park). + */ + if (!test_bit(RBIO_SYNC_PARK_BIT, &rbio->flags) && + time_before(jiffies, rbio->park_stuck_deadline) && + !parked_rbio_ready(rbio)) { + rearm = true; + continue; + } rbio_unpark_locked(rbio); list_add_tail(&rbio->parked_node, &flush); } spin_unlock(&table->parked_lock); list_for_each_entry_safe(rbio, tmp, &flush, parked_node) { + struct btrfs_fs_info *ffs = rbio->bioc->fs_info; + list_del_init(&rbio->parked_node); - atomic64_inc(&rbio->bioc->fs_info->stripe_park_stats.expired); + if (!test_bit(RBIO_SYNC_PARK_BIT, &rbio->flags) && + time_after_eq(jiffies, rbio->park_stuck_deadline)) + atomic64_inc(&ffs->stripe_park_stats.stuck); + else + atomic64_inc(&ffs->stripe_park_stats.expired); start_async_work(rbio, rmw_rbio_work_locked); } if (rearm) @@ -1075,9 +1142,21 @@ static noinline int lock_stripe_add(struct btrfs_raid_bio *rbio) * this stripe. */ if (test_bit(RBIO_PARKED_BIT, &cur->flags)) { + struct btrfs_fs_info *cfs = + cur->bioc->fs_info; + if (cur->bio_list_bytes == cur->nr_data * BTRFS_STRIPE_LEN) { - unpark_full_rbio(cur); + unpark_ready_rbio(cur, + &cfs->stripe_park_stats.filled); + } else if (parked_rbio_ready(cur)) { + /* + * This merge completed the allocated + * prefix; the rest of the stripe is + * dead space the pad fills. + */ + unpark_ready_rbio(cur, + &cfs->stripe_park_stats.unparked_ready); } else if (sync) { struct btrfs_stripe_hash_table *table = cur->bioc->fs_info->stripe_hash_table; @@ -1087,9 +1166,13 @@ static noinline int lock_stripe_add(struct btrfs_raid_bio *rbio) spin_lock(&table->parked_lock); if (test_bit(RBIO_PARKED_BIT, - &cur->flags) && - time_before(dl, cur->park_deadline)) - cur->park_deadline = dl; + &cur->flags)) { + set_bit(RBIO_SYNC_PARK_BIT, + &cur->flags); + if (time_before(dl, + cur->park_deadline)) + cur->park_deadline = dl; + } spin_unlock(&table->parked_lock); } } diff --git a/fs/btrfs/raid56.h b/fs/btrfs/raid56.h index 07ba71ae8e54f..ca15c45f8f4be 100644 --- a/fs/btrfs/raid56.h +++ b/fs/btrfs/raid56.h @@ -140,6 +140,8 @@ struct btrfs_raid_bio { */ struct list_head parked_node; unsigned long park_deadline; + /* Hard cap on completion-waiting (see parked_rbio_ready()). */ + unsigned long park_stuck_deadline; /* Flags that tell us if it is safe to merge with this bio. */ unsigned long flags; @@ -297,6 +299,6 @@ void raid56_parity_cache_data_folios(struct btrfs_raid_bio *rbio, int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info); void btrfs_free_stripe_hash_table(struct btrfs_fs_info *info); void btrfs_flush_parked_rbios(struct btrfs_fs_info *fs_info, u64 start, - u64 num_bytes); + u64 num_bytes, bool force); #endif diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c index 06ca412ae5687..9b98cf58b366c 100644 --- a/fs/btrfs/sysfs.c +++ b/fs/btrfs/sysfs.c @@ -1122,8 +1122,10 @@ static ssize_t btrfs_stripe_park_stats_show(struct kobject *kobj, "parked %lld\n" "merged_bytes %lld\n" "filled %lld\n" + "unparked_ready %lld\n" "kicked %lld\n" "expired %lld\n" + "stuck %lld\n" "padded %lld\n" "rmw_reads %lld\n" "data_rmw %lld\n" @@ -1138,8 +1140,10 @@ static ssize_t btrfs_stripe_park_stats_show(struct kobject *kobj, atomic64_read(&fs_info->stripe_park_stats.parked), atomic64_read(&fs_info->stripe_park_stats.merged_bytes), atomic64_read(&fs_info->stripe_park_stats.filled), + atomic64_read(&fs_info->stripe_park_stats.unparked_ready), atomic64_read(&fs_info->stripe_park_stats.kicked), atomic64_read(&fs_info->stripe_park_stats.expired), + atomic64_read(&fs_info->stripe_park_stats.stuck), atomic64_read(&fs_info->stripe_park_stats.padded), atomic64_read(&fs_info->stripe_park_stats.rmw_reads), atomic64_read(&fs_info->stripe_park_stats.data_rmw),