From 73b609ea6a4c6cf0020cb27f3a591c8b0e17ba08 Mon Sep 17 00:00:00 2001 From: Zygo Blaxell Date: Tue, 28 Jul 2026 11:19:50 -0400 Subject: [PATCH] btrfs: raid56: batch stripe_alloc partial writes into full-stripe writes A raid56 sub-stripe write pays a read-modify-write: read the rest of the stripe, recompute parity, write. The plug callback merges rbios submitted within one plug window and the stripe cache saves re-reads for back-to-back writes, but measurement shows what survives them: on a 4-device raid5, small-file writeback produces 100% sub-stripe rbios (zero full-stripe writes) and a mixed 4K-512K workload ~55%, so parity is recomputed and rewritten once per file that lands in a stripe. Stripe-exclusive allocation gives raid56 something the allocator never guaranteed before: the rest of an open run's stripe is either written before the run closes -- within the current transaction -- or not at all. That makes holding a partial write a sound bet, so park it: a partial write rbio whose stripe belongs to a stripe run keeps the stripe lock but does not start its RMW. Later writes to the stripe merge into it through the existing lock_stripe_add() path, and the moment its data bitmap covers the stripe it is submitted as a single full-stripe write: one parity computation, no reads. A draining run counts, not just an open one. A run closes the moment it is fully allocated -- typically milliseconds before its last writes reach the raid56 layer -- so gating on "open" would leave exactly those tail writes unable to park and unable to pad, and they are the bulk of the read-modify-writes this patch exists to remove. A draining run is in fact the safest thing to park against: its frontier is frozen, so everything below it is inflight IO the commit drain already waits for (arrival guaranteed) and everything above it is dead space the pad may fill. Completion accounting can never trigger the flush of runts -- parked bios have not been submitted, and preallocated or discarded allocations produce no bios at all -- so liveness comes from explicit flushes: the run retirement paths flush before the commit's IO drain (which waits on the very bios parked here), ordered-extent waiters flush their range on a short retry period, and a timer bounds everything else. Writes with a blocked waiter (REQ_SYNC: fsync- and sync-driven writeback) park with a ~3ms deadline instead of 100ms -- long enough for the rest of one writeback pass over the stripe to merge in, short enough to stay invisible in fsync latency -- and a sync bio merging into a parked rbio pulls the deadline in. Only the bg->open_stripe slots are consulted for eligibility, and a parked rbio that a racing retirement closed out from under is bounded by the timer, never by the drain. Measured (same 4-device raid5, deterministic workloads, classifying every write rbio at rmw_rbio): workload full-stripe rbios total write rbios streaming 99% -> 99% unchanged mixed 4K-512K 43% -> 97% 2478 -> 1615 small files 0% -> 86% 1537 -> 335 (stripe reads 311->23) fsync-per-file 0% -> 0% unchanged (nothing co-pending) fsync latency is 12ms vs 8ms unpatched (each fsync pays one short park). Batching also collapses trapped free space in a mixed-size forward fill from ~87 MiB to ~3 MiB: stripes now fill completely before the commit closes their run, so the frontier strands almost nothing. Validated: btrfs selftests; mixed-size + concurrency + churn + balance workload with trapped-space accounting returning to zero after deleting everything; scrub and full sha256 read-back over 4005 mixed random files; and the dm-log-writes write-hole crash matrix (24/24 controls clean, 0/36 armed cases with committed data damage) -- parking does not perturb the stripe-exclusivity guarantee it rides on. Assisted-by: Claude:claude-fable-5 --- fs/btrfs/block-group.c | 44 ++++++++ fs/btrfs/block-group.h | 1 + fs/btrfs/ordered-data.c | 22 ++++ fs/btrfs/raid56.c | 237 +++++++++++++++++++++++++++++++++++++++- fs/btrfs/raid56.h | 11 ++ 5 files changed, 314 insertions(+), 1 deletion(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 107a9a82606e4..3856e129a8308 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -742,6 +742,40 @@ out: return ret; } +/* + * Does @logical lie within a stripe run (open or draining)? Used by the + * raid56 layer to decide whether a partial write to this stripe may be + * parked to collect into a full-stripe write: an open run guarantees the + * rest of the stripe is either filled before the run closes or never, and + * a draining run's frozen frontier makes "never" decidable -- everything + * allocated below it is inflight IO the commit drain already waits for, + * and everything above it is dead space the pad fills. Excluding draining + * runs made every run's tail writes unparkable -- a run closes on + * exhaustion in the gap between allocation and write arrival -- and those + * writes are the bulk of stripe_alloc's read-modify-writes. + */ +bool btrfs_stripe_in_open_run(struct btrfs_fs_info *fs_info, u64 logical) +{ + struct btrfs_block_group *bg; + struct btrfs_open_stripe_run *run; + unsigned long flags; + bool ret = false; + + bg = btrfs_lookup_block_group(fs_info, logical); + if (!bg) + return false; + spin_lock_irqsave(&bg->stripe_run_lock, flags); + list_for_each_entry(run, &bg->open_stripe_runs, list) { + if (logical >= run->start && logical < run->end) { + ret = true; + break; + } + } + spin_unlock_irqrestore(&bg->stripe_run_lock, flags); + btrfs_put_block_group(bg); + return ret; +} + /* * Report completed (or abandoned) data IO for an allocation made with * btrfs_alloc_from_open_stripe(). Every allocated byte must be reported @@ -911,6 +945,8 @@ static bool bg_open_stripes_settled(struct btrfs_block_group *bg, u64 seq) void btrfs_retire_block_group_stripes(struct btrfs_block_group *bg) { close_block_group_stripe_runs(bg, U64_MAX); + /* Parked partial-stripe rbios hold the very bios the wait drains. */ + btrfs_flush_parked_rbios(bg->fs_info, bg->start, bg->length); wait_var_event(&bg->open_stripe_runs, bg_open_stripes_settled(bg, U64_MAX)); } @@ -946,6 +982,14 @@ void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info, list_for_each_entry(bg, &retire_list, open_stripe_retire_list) close_block_group_stripe_runs(bg, seq); + /* + * Parked partial-stripe write rbios (see raid56.c) hold bios whose + * write_done the drain below waits for; kick them all down first. + * A racing park that saw its run still open lands after this flush + * and is bounded by the park timer, not by us. + */ + btrfs_flush_parked_rbios(fs_info, 0, U64_MAX); + while (!list_empty(&retire_list)) { bg = list_first_entry(&retire_list, struct btrfs_block_group, open_stripe_retire_list); diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 69f4546316be0..ebcae80cde7f9 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -392,6 +392,7 @@ struct btrfs_open_stripe_run *btrfs_get_open_stripe_run( u64 *open_seq); void btrfs_close_bg_open_stripes(struct btrfs_block_group *bg); void btrfs_retire_block_group_stripes(struct btrfs_block_group *bg); +bool btrfs_stripe_in_open_run(struct btrfs_fs_info *fs_info, u64 logical); void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info, struct btrfs_transaction *trans); void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg); diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index b5307c3345b57..3c168034f15ff 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -20,6 +20,7 @@ #include "subpage.h" #include "file.h" #include "block-group.h" +#include "raid56.h" static struct kmem_cache *btrfs_ordered_extent_cache; @@ -927,6 +928,27 @@ void btrfs_start_ordered_extent_nowriteback(struct btrfs_ordered_extent *entry, if (!freespace_inode) btrfs_might_wait_for_event(inode->root->fs_info, btrfs_ordered_extent); + /* + * A parked partial-stripe write rbio (raid56 full-stripe batching) may + * be holding this extent's bios; kick it down or the wait is bounded + * by the park timer instead of the IO. The park happens on a worker + * after the submission above returns, so a single flush can run too + * early and miss it -- retry on a short period until the extent + * completes. Only stripe_alloc filesystems ever park, so everything + * else takes the plain single sleep below. + */ + while (btrfs_test_opt(inode->root->fs_info, STRIPE_ALLOC) && + entry->disk_num_bytes && + !test_bit(BTRFS_ORDERED_COMPLETE, &entry->flags)) { + btrfs_flush_parked_rbios(inode->root->fs_info, + entry->disk_bytenr, + entry->disk_num_bytes); + if (wait_event_timeout(entry->wait, + test_bit(BTRFS_ORDERED_COMPLETE, + &entry->flags), + msecs_to_jiffies(10))) + break; + } wait_event(entry->wait, test_bit(BTRFS_ORDERED_COMPLETE, &entry->flags)); } diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index cabb7939e6787..756af2b49d457 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -37,8 +37,28 @@ */ #define RBIO_CACHE_READY_BIT 3 +/* + * Set while a partial write rbio is parked: it owns the stripe lock but its + * RMW has not been started, so writes filling the rest of an open stripe + * run's stripe merge into it (see rbio_try_park()). Whoever clears this bit + * owns starting the rbio's work, exactly once. + */ +#define RBIO_PARKED_BIT 4 + #define RBIO_CACHE_SIZE 1024 +/* + * How long a parked partial write rbio may wait for merges before flushing. + * Sync writes (fsync- or sync-driven writeback, REQ_SYNC) have a waiter + * blocked on their completion, so they get a much shorter deadline: long + * enough for the rest of one writeback pass over the same stripe to merge + * in, short enough to stay invisible in fsync latency. + */ +#define BTRFS_RBIO_PARK_TIMEOUT_MS 100 +#define BTRFS_RBIO_PARK_SYNC_TIMEOUT_MS 3 +/* Scan period of the park timer while anything is parked. */ +#define BTRFS_RBIO_PARK_SCAN_MS 3 + #define BTRFS_STRIPE_HASH_TABLE_BITS 11 static void dump_bioc(const struct btrfs_fs_info *fs_info, const struct btrfs_io_context *bioc) @@ -131,6 +151,14 @@ struct btrfs_stripe_hash_table { struct list_head stripe_cache; spinlock_t cache_lock; int cache_size; + /* + * Parked partial write rbios (RBIO_PARKED_BIT), in park order. + * parked_lock nests inside the hash and bio_list locks. + */ + spinlock_t parked_lock; + struct list_head parked; + struct delayed_work parked_work; + struct btrfs_fs_info *fs_info; struct btrfs_stripe_hash table[]; }; @@ -150,6 +178,7 @@ struct sector_ptr { static void rmw_rbio_work(struct work_struct *work); static void rmw_rbio_work_locked(struct work_struct *work); +static void parked_rbios_timeout_work(struct work_struct *work); static void index_rbio_pages(struct btrfs_raid_bio *rbio); static int alloc_rbio_pages(struct btrfs_raid_bio *rbio); @@ -222,6 +251,10 @@ int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info) spin_lock_init(&table->cache_lock); INIT_LIST_HEAD(&table->stripe_cache); + spin_lock_init(&table->parked_lock); + INIT_LIST_HEAD(&table->parked); + INIT_DELAYED_WORK(&table->parked_work, parked_rbios_timeout_work); + table->fs_info = info; h = table->table; @@ -534,6 +567,8 @@ void btrfs_free_stripe_hash_table(struct btrfs_fs_info *info) { if (!info->stripe_hash_table) return; + cancel_delayed_work_sync(&info->stripe_hash_table->parked_work); + WARN_ON(!list_empty(&info->stripe_hash_table->parked)); btrfs_clear_rbio_cache(info); kvfree(info->stripe_hash_table); info->stripe_hash_table = NULL; @@ -678,6 +713,174 @@ static int rbio_can_merge(struct btrfs_raid_bio *last, return 1; } +/* + * Full-stripe write batching for stripe-exclusive allocation. + * + * A partial write rbio whose stripe belongs to an open stripe run is parked: + * it keeps the stripe lock but its RMW is not started, so the writes that + * fill the rest of the stripe -- guaranteed to arrive before the run closes, + * or not at all -- merge into it through lock_stripe_add() instead of each + * paying a read-modify-write. The moment the data bitmap fills, the rbio is + * submitted as a single full-stripe write: one parity computation, no reads. + * + * Completion accounting can never be the flush trigger -- parked bios have + * not been submitted, and preallocated or discarded allocations produce no + * bios at all -- so everything that does not fill is flushed by the run + * retirement paths (before the commit's IO drain, which waits on the very + * bios parked here), by ordered-extent waiters, and by a timer that bounds + * both latency and the memory the parked pages pin. + */ + +/* Caller holds table->parked_lock. */ +static void rbio_unpark_locked(struct btrfs_raid_bio *rbio) +{ + list_del_init(&rbio->parked_node); + clear_bit(RBIO_PARKED_BIT, &rbio->flags); +} + +/* Does any bio gathered in @rbio carry a sync hint (a blocked waiter)? */ +static bool rbio_has_sync_bio(struct btrfs_raid_bio *rbio) +{ + struct bio *bio; + + bio_list_for_each(bio, &rbio->bio_list) + if (bio->bi_opf & REQ_SYNC) + return true; + return false; +} + +/* + * Park a partial write rbio that owns its stripe lock, if its stripe is + * covered by an open stripe run. Returns true if the rbio was parked (or + * concurrently filled and requeued); false if the caller should proceed + * with the RMW itself. + */ +static bool rbio_try_park(struct btrfs_raid_bio *rbio) +{ + struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; + struct btrfs_stripe_hash_table *table = fs_info->stripe_hash_table; + unsigned int timeout_ms = BTRFS_RBIO_PARK_TIMEOUT_MS; + + if (!btrfs_stripe_in_open_run(fs_info, rbio->bioc->full_stripe_logical)) + return false; + + spin_lock(&rbio->bio_list_lock); + if (rbio_has_sync_bio(rbio)) + timeout_ms = BTRFS_RBIO_PARK_SYNC_TIMEOUT_MS; + spin_unlock(&rbio->bio_list_lock); + + spin_lock(&table->parked_lock); + set_bit(RBIO_PARKED_BIT, &rbio->flags); + rbio->park_deadline = jiffies + msecs_to_jiffies(timeout_ms); + list_add_tail(&rbio->parked_node, &table->parked); + spin_unlock(&table->parked_lock); + queue_delayed_work(system_percpu_wq, &table->parked_work, + msecs_to_jiffies(BTRFS_RBIO_PARK_SCAN_MS)); + + /* + * A merge may have filled us between the caller's fullness check and + * the park above; reclaim the park so the full stripe goes straight + * down. If the merger's unpark won instead, it queued the work. + */ + if (rbio_is_full(rbio)) { + spin_lock(&table->parked_lock); + if (test_bit(RBIO_PARKED_BIT, &rbio->flags)) { + rbio_unpark_locked(rbio); + spin_unlock(&table->parked_lock); + return false; + } + spin_unlock(&table->parked_lock); + } + return true; +} + +/* + * A merge just filled a parked rbio: unpark it and start its (now read-free) + * write. Callers may hold the stripe hash and bio_list locks; parked_lock + * nests inside both. + */ +static void unpark_full_rbio(struct btrfs_raid_bio *rbio) +{ + struct btrfs_stripe_hash_table *table = + rbio->bioc->fs_info->stripe_hash_table; + + spin_lock(&table->parked_lock); + if (!test_bit(RBIO_PARKED_BIT, &rbio->flags)) { + spin_unlock(&table->parked_lock); + return; + } + rbio_unpark_locked(rbio); + spin_unlock(&table->parked_lock); + start_async_work(rbio, rmw_rbio_work_locked); +} + +/* + * Flush every parked rbio overlapping [start, start + num_bytes) down its + * normal RMW path. Called by run retirement (before the commit's IO drain), + * ordered-extent waiters, and the park timer. + */ +void btrfs_flush_parked_rbios(struct btrfs_fs_info *fs_info, u64 start, + u64 num_bytes) +{ + struct btrfs_stripe_hash_table *table = fs_info->stripe_hash_table; + struct btrfs_raid_bio *rbio; + struct btrfs_raid_bio *tmp; + LIST_HEAD(flush); + + if (!table || list_empty_careful(&table->parked)) + return; + spin_lock(&table->parked_lock); + list_for_each_entry_safe(rbio, tmp, &table->parked, parked_node) { + u64 rbio_start = rbio->bioc->full_stripe_logical; + u64 rbio_len = (u64)rbio->nr_data * BTRFS_STRIPE_LEN; + + if (rbio_start + rbio_len <= start || + (num_bytes != U64_MAX && rbio_start >= start + num_bytes)) + continue; + rbio_unpark_locked(rbio); + list_add_tail(&rbio->parked_node, &flush); + } + spin_unlock(&table->parked_lock); + + list_for_each_entry_safe(rbio, tmp, &flush, parked_node) { + list_del_init(&rbio->parked_node); + start_async_work(rbio, rmw_rbio_work_locked); + } +} + +/* Flush parked rbios whose deadline has passed; re-arm while any remain. */ +static void parked_rbios_timeout_work(struct work_struct *work) +{ + struct btrfs_stripe_hash_table *table = + container_of(work, struct btrfs_stripe_hash_table, + parked_work.work); + struct btrfs_raid_bio *rbio; + struct btrfs_raid_bio *tmp; + LIST_HEAD(flush); + bool rearm = false; + + spin_lock(&table->parked_lock); + list_for_each_entry_safe(rbio, tmp, &table->parked, parked_node) { + /* Deadlines are not ordered (sync parks are shorter). */ + if (time_before(jiffies, rbio->park_deadline)) { + rearm = true; + continue; + } + rbio_unpark_locked(rbio); + list_add_tail(&rbio->parked_node, &flush); + } + spin_unlock(&table->parked_lock); + + list_for_each_entry_safe(rbio, tmp, &flush, parked_node) { + list_del_init(&rbio->parked_node); + start_async_work(rbio, rmw_rbio_work_locked); + } + if (rearm) + queue_delayed_work(system_percpu_wq, &table->parked_work, + msecs_to_jiffies(BTRFS_RBIO_PARK_SCAN_MS)); +} + +/* Return the sector index for @stripe_nr and @sector_nr. */ static unsigned int rbio_stripe_sector_index(const struct btrfs_raid_bio *rbio, unsigned int stripe_nr, unsigned int sector_nr) @@ -770,7 +973,35 @@ static noinline int lock_stripe_add(struct btrfs_raid_bio *rbio) /* Can we merge into the lock owner? */ if (rbio_can_merge(cur, rbio)) { + bool sync = rbio_has_sync_bio(rbio); + merge_rbio(cur, rbio); + /* + * Our bios may have completed a parked stripe. We + * hold cur->bio_list_lock, so test fullness directly + * (rbio_is_full() would self-deadlock on it). A sync + * bio merging into a still-partial parked rbio pulls + * its flush deadline in: a waiter is now blocked on + * this stripe. + */ + if (test_bit(RBIO_PARKED_BIT, &cur->flags)) { + if (cur->bio_list_bytes == + cur->nr_data * BTRFS_STRIPE_LEN) { + unpark_full_rbio(cur); + } else if (sync) { + struct btrfs_stripe_hash_table *table = + cur->bioc->fs_info->stripe_hash_table; + unsigned long dl = jiffies + + msecs_to_jiffies(BTRFS_RBIO_PARK_SYNC_TIMEOUT_MS); + + spin_lock(&table->parked_lock); + if (test_bit(RBIO_PARKED_BIT, + &cur->flags) && + time_before(dl, cur->park_deadline)) + cur->park_deadline = dl; + spin_unlock(&table->parked_lock); + } + } spin_unlock(&cur->bio_list_lock); freeit = rbio; ret = 1; @@ -1040,6 +1271,7 @@ static struct btrfs_raid_bio *alloc_rbio(struct btrfs_fs_info *fs_info, spin_lock_init(&rbio->bio_list_lock); INIT_LIST_HEAD(&rbio->stripe_cache); INIT_LIST_HEAD(&rbio->hash_list); + INIT_LIST_HEAD(&rbio->parked_node); btrfs_get_bioc(bioc); rbio->bioc = bioc; rbio->nr_pages = num_pages; @@ -2426,8 +2658,11 @@ static void rmw_rbio_work(struct work_struct *work) struct btrfs_raid_bio *rbio; rbio = container_of(work, struct btrfs_raid_bio, work); - if (lock_stripe_add(rbio) == 0) + if (lock_stripe_add(rbio) == 0) { + if (!rbio_is_full(rbio) && rbio_try_park(rbio)) + return; rmw_rbio(rbio); + } } static void rmw_rbio_work_locked(struct work_struct *work) diff --git a/fs/btrfs/raid56.h b/fs/btrfs/raid56.h index 84c4d1d29c7a8..5e8887169696c 100644 --- a/fs/btrfs/raid56.h +++ b/fs/btrfs/raid56.h @@ -55,6 +55,15 @@ struct btrfs_raid_bio { */ struct list_head plug_list; + /* + * Membership in the stripe hash table's parked list: partial write + * rbios for stripes of open stripe runs (stripe_alloc) are held back, + * collecting merges until they can go down as one full-stripe write. + * Protected by the table's parked_lock. + */ + struct list_head parked_node; + unsigned long park_deadline; + /* Flags that tell us if it is safe to merge with this bio. */ unsigned long flags; @@ -206,5 +215,7 @@ void raid56_parity_cache_data_folios(struct btrfs_raid_bio *rbio, int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info); void btrfs_free_stripe_hash_table(struct btrfs_fs_info *info); +void btrfs_flush_parked_rbios(struct btrfs_fs_info *fs_info, u64 start, + u64 num_bytes); #endif -- 2.53.0