* closes the raid56 write hole for these block groups.
*
* Each run counts the bytes reserved from it whose data IO has not yet
- * completed (inflight_bytes). At commit time, once the transaction can
- * no longer accept joins (TRANS_STATE_COMMIT_DOING with a single writer
- * left), btrfs_retire_open_stripes() closes every run opened before the
- * commit and waits for their inflight bytes to drain. At that point
- * every extent the committing transaction references sits in a stripe
- * that will never be written again, and every write to such a stripe has
- * reached disk before the superblock is written. This is a pure data-IO
- * wait: it must never wait for ordered extent *completion*, whose
- * processing needs a transaction join that is blocked at this stage.
+ * completed (inflight_bytes). At commit time, while the transaction
+ * still accepts joins (before TRANS_STATE_COMMIT_DOING),
+ * btrfs_retire_open_stripes() closes every run opened before the commit
+ * and waits for their inflight bytes to drain. The placement matters
+ * twice over. It must be a pure data-IO wait, never an ordered extent
+ * completion wait (whose processing needs a transaction join). And it
+ * must run where joins still succeed: a task holding unreported run
+ * bytes -- a reserved extent whose bio has not completed -- may block on
+ * a transaction join, e.g. for a chunk allocation while walking a
+ * delalloc range, and the drain waits for those very bytes.
*
* Allocations racing with the commit open new runs stamped with a newer
- * open_seq and are not retired or waited for: metadata referencing them
- * can only join the next transaction, so they belong to the next window.
- * The retire walk may only rely on this for allocations whose block group
- * membership (fs_info->open_stripe_bgs) was established before the walk
- * collected its worklist; btrfs_alloc_from_open_stripe() therefore adds
- * the membership before returning, i.e. before the caller can create the
- * ordered extent for the allocation.
+ * open_seq and are not retired or waited for: they belong to the next
+ * window. Because the commit still accepts joins after the drain, an
+ * ordered extent from such a run could complete fast and try to add its
+ * file extent to the committing transaction; the finish path compares
+ * the extent's run sequence with the transaction's retired sequence and
+ * defers to the next transaction instead. The membership add, the
+ * sequence read and the run installation happen atomically under
+ * fs_info->open_stripe_lock against the walk's bump-and-snapshot, so
+ * every run is either seen by the walk that retires its window or
+ * stamped with a sequence that forces the deferral.
*
* Lock order: fs_info->open_stripe_lock outside bg->stripe_run_lock. The
* stripe_run_lock is irq-safe because completed data IO is reported from
return tail_len;
}
-static void open_stripe_add_bg(struct btrfs_fs_info *fs_info,
- struct btrfs_block_group *bg)
-{
- spin_lock(&fs_info->open_stripe_lock);
- if (list_empty(&bg->open_stripe_bg_list)) {
- btrfs_get_block_group(bg);
- list_add_tail(&bg->open_stripe_bg_list,
- &fs_info->open_stripe_bgs);
- }
- spin_unlock(&fs_info->open_stripe_lock);
-}
-
/*
* Allocate num_bytes from the block group's open stripe run, opening a new
* run via btrfs_claim_free_stripe_run() when there is none or the current
goto out;
}
+ /*
+ * Install under fs_info->open_stripe_lock so that the block group
+ * membership, the retire sequence read and the run installation are
+ * one atomic step against the commit's retire walk (which bumps the
+ * sequence and snapshots the membership list under the same lock).
+ * A run is therefore either visible to the walk that retires its
+ * window, or stamped with a sequence that walk's transaction did
+ * not retire, which defers its extents to the next transaction.
+ */
+ spin_lock(&fs_info->open_stripe_lock);
+ if (list_empty(&bg->open_stripe_bg_list)) {
+ btrfs_get_block_group(bg);
+ list_add_tail(&bg->open_stripe_bg_list,
+ &fs_info->open_stripe_bgs);
+ }
spin_lock_irqsave(&bg->stripe_run_lock, flags);
if (bg->ro) {
/*
* must not be installed.
*/
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ spin_unlock(&fs_info->open_stripe_lock);
btrfs_add_free_space(bg, start, len);
*available = 0;
ret = -ENOSPC;
new_run->end = start + len;
new_run->offset = start + num_bytes;
new_run->inflight_bytes = num_bytes;
- new_run->open_seq = READ_ONCE(fs_info->stripe_retire_seq);
+ new_run->open_seq = fs_info->stripe_retire_seq;
new_run->open = (new_run->offset != new_run->end);
list_add_tail(&new_run->list, &bg->open_stripe_runs);
bg->open_stripe[class] = new_run->open ? new_run : NULL;
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ spin_unlock(&fs_info->open_stripe_lock);
if (tail_len)
btrfs_add_free_space(bg, tail_start, tail_len);
*ret_offset = start;
new_run = NULL;
ret = 0;
- /* Must precede the caller's ordered extent creation; see above. */
- open_stripe_add_bg(fs_info, bg);
out:
kfree(new_run);
return ret;
* with inflight bytes is never freed.
*/
struct btrfs_open_stripe_run *btrfs_get_open_stripe_run(
- struct btrfs_fs_info *fs_info, u64 bytenr, u64 num_bytes)
+ struct btrfs_fs_info *fs_info, u64 bytenr, u64 num_bytes,
+ u64 *open_seq)
{
struct btrfs_open_stripe_run *run = NULL;
struct btrfs_open_stripe_run *iter;
ASSERT(bytenr + num_bytes <= iter->offset);
ASSERT(iter->inflight_bytes >= num_bytes);
run = iter;
+ *open_seq = iter->open_seq;
break;
}
}
/*
* Commit-time retirement: close every stripe run opened before this call
* and wait for all data IO into the stripes of those runs to complete.
- * Must run where the committing transaction can no longer accept joins
- * (TRANS_STATE_COMMIT_DOING, single writer); see the comment at the top.
- * Calls are serialized by the transaction commit.
+ * Must run while the committing transaction still accepts joins (before
+ * TRANS_STATE_COMMIT_DOING): tasks holding unreported run bytes can block
+ * on a join (e.g. a chunk allocation mid-delalloc), and this drain waits
+ * for their IO. Records the bumped sequence in @trans; ordered extents
+ * from runs with open_seq >= it defer their file extents to the next
+ * transaction, closing the fast-completion window this placement opens.
+ * Calls are serialized by the transaction commit; @trans may be NULL for
+ * the final cleanup call.
*/
-void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info)
+void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info,
+ struct btrfs_transaction *trans)
{
struct btrfs_block_group *bg;
LIST_HEAD(retire_list);
spin_lock(&fs_info->open_stripe_lock);
seq = ++fs_info->stripe_retire_seq;
+ if (trans)
+ trans->stripe_retire_seq = seq;
list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
list_add_tail(&bg->open_stripe_retire_list, &retire_list);
spin_unlock(&fs_info->open_stripe_lock);
}
}
+/*
+ * With stripe-exclusive allocation, no in-place (nocow/prealloc) write may
+ * land in a raid56 data block group: it could tear a stripe containing
+ * committed extents of other files, which is exactly the write hole. The
+ * write paths use this to force COW for such extents.
+ */
+bool btrfs_stripe_alloc_forces_cow(struct btrfs_fs_info *fs_info, u64 bytenr)
+{
+ struct btrfs_block_group *bg;
+ bool ret = false;
+
+ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+ return false;
+ bg = btrfs_lookup_block_group(fs_info, bytenr);
+ if (bg) {
+ ret = btrfs_is_stripe_alloc_bg(bg);
+ btrfs_put_block_group(bg);
+ }
+ return ret;
+}
+
/*
* Drop the dedication of a block group to data relocation. Shared by the
* zoned allocator and the stripe allocation policy; both dedicate one
struct btrfs_caching_control *caching_ctl;
struct rb_node *n;
+ /*
+ * The final commit retired all stripe runs; this only matters after
+ * a transaction abort, where ordered extent teardown has reported
+ * all inflight IO, so the drain cannot block. It releases the run
+ * memory and the block group references held by the membership list.
+ */
+ btrfs_retire_open_stripes(info, NULL);
+
if (btrfs_is_zoned(info)) {
if (info->active_meta_bg) {
btrfs_put_block_group(info->active_meta_bg);
void btrfs_open_stripe_write_done_bytenr(struct btrfs_fs_info *fs_info,
u64 start, u64 num_bytes);
struct btrfs_open_stripe_run *btrfs_get_open_stripe_run(
- struct btrfs_fs_info *fs_info, u64 bytenr, u64 num_bytes);
+ struct btrfs_fs_info *fs_info, u64 bytenr, u64 num_bytes,
+ u64 *open_seq);
void btrfs_close_bg_open_stripes(struct btrfs_block_group *bg);
void btrfs_retire_block_group_stripes(struct btrfs_block_group *bg);
-void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info);
+void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info,
+ struct btrfs_transaction *trans);
void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg);
void btrfs_release_data_reloc_bg(struct btrfs_fs_info *fs_info);
+bool btrfs_stripe_alloc_forces_cow(struct btrfs_fs_info *fs_info, u64 bytenr);
+bool btrfs_is_stripe_alloc_bg(const struct btrfs_block_group *bg);
struct btrfs_block_group *btrfs_inc_nocow_writers(struct btrfs_fs_info *fs_info,
u64 bytenr);
void btrfs_dec_nocow_writers(struct btrfs_block_group *bg);
* sub-stripe writes into stripes containing committed data is what closes
* the raid56 write hole for datacow writes; see the "Open stripe runs"
* comment in block-group.c.
+ *
+ * Data only, and not the data in a mixed block group: metadata is not ready
+ * for this allocator (see the mixed block group rejection in
+ * btrfs_check_mountopts(), which is the real gate -- the METADATA test here
+ * is belt and braces for a block group that somehow reaches this path).
*/
-static bool btrfs_is_stripe_alloc_bg(const struct btrfs_block_group *bg)
+bool btrfs_is_stripe_alloc_bg(const struct btrfs_block_group *bg)
{
return btrfs_test_opt(bg->fs_info, STRIPE_ALLOC) &&
(bg->flags & BTRFS_BLOCK_GROUP_DATA) &&
+ !(bg->flags & BTRFS_BLOCK_GROUP_METADATA) &&
(bg->flags & BTRFS_BLOCK_GROUP_RAID56_MASK);
}
args->file_extent.offset += args->start - key->offset;
io_start = args->file_extent.disk_bytenr + args->file_extent.offset;
+ /*
+ * With stripe-exclusive allocation, an in-place write into a raid56
+ * data block group would reintroduce the write hole for every other
+ * extent in the target stripe, so force COW for such extents. The
+ * data relocation inode is exempt: its preallocated extents live in
+ * relocation-class stripe runs that never share stripes with other
+ * data, and relocation depends on its writes landing in place.
+ */
+ if (!btrfs_is_data_reloc_root(root) &&
+ btrfs_stripe_alloc_forces_cow(root->fs_info, io_start))
+ goto out;
+
/*
* Force COW if csums exist in the range. This ensures that csums for a
* given extent are either valid or do not exist.
goto out;
}
+ /*
+ * If this extent came from a stripe run window that the joined
+ * transaction's commit did not retire, its file extent must not be
+ * added to that transaction: the extent's stripe was not drained
+ * before the superblock write, and a torn write after the commit
+ * could damage it. Wait for the commit to unblock and join again,
+ * attaching to a later transaction (which will retire the window).
+ */
+ while (ordered_extent->stripe_seq) {
+ u64 transid = trans->transid;
+ u64 retired;
+
+ spin_lock(&fs_info->open_stripe_lock);
+ retired = trans->transaction->stripe_retire_seq;
+ spin_unlock(&fs_info->open_stripe_lock);
+ if (!retired || ordered_extent->stripe_seq - 1 < retired)
+ break;
+ btrfs_end_transaction(trans);
+ /*
+ * The wait below sleeps until the commit's critical section
+ * ends, and must not hold the extent lock across that:
+ * buffered writers block on this range while holding their
+ * prepared locked folios, which strands writeback, reclaim
+ * and compaction -- resources the committing transaction's
+ * stripe drain can depend on (via the data bios it waits
+ * for). Dropping the lock is safe here: the range is still
+ * covered by this pending ordered extent, so any writer that
+ * takes the lock finds the ordered extent and waits for it
+ * without touching the range.
+ */
+ if (clear_bits & EXTENT_LOCKED)
+ btrfs_clear_extent_bit(io_tree, start, end,
+ EXTENT_LOCKED |
+ EXTENT_FINISHING_ORDERED,
+ &cached_state);
+ btrfs_wait_transid_unblocked(fs_info, transid);
+ if (clear_bits & EXTENT_LOCKED)
+ btrfs_lock_extent_bits(io_tree, start, end,
+ EXTENT_LOCKED |
+ EXTENT_FINISHING_ORDERED,
+ &cached_state);
+ trans = btrfs_join_transaction(root);
+ if (IS_ERR(trans)) {
+ ret = PTR_ERR(trans);
+ trans = NULL;
+ goto out;
+ }
+ }
+
trans->block_rsv = &inode->block_rsv;
ret = btrfs_insert_raid_extent(trans, ordered_extent);
* stripe run (runs are claimed from fully-free stripes).
*/
if (!is_nocow &&
- !list_empty_careful(&inode->root->fs_info->open_stripe_bgs))
+ !list_empty_careful(&inode->root->fs_info->open_stripe_bgs)) {
+ u64 open_seq;
+
entry->stripe_run = btrfs_get_open_stripe_run(
inode->root->fs_info, disk_bytenr,
- disk_num_bytes);
+ disk_num_bytes, &open_seq);
+ if (entry->stripe_run)
+ entry->stripe_seq = open_seq + 1;
+ }
INIT_LIST_HEAD(&entry->root_extent_list);
INIT_LIST_HEAD(&entry->work_list);
INIT_LIST_HEAD(&entry->bioc_list);
* The open stripe run backing this extent's disk range, if it was
* allocated by the raid56 stripe allocation policy; this extent's
* data IO is reported to it exactly once (at IO completion, or when
- * the ordered extent is freed without completing IO).
+ * the ordered extent is freed without completing IO). stripe_seq
+ * is the run's open_seq + 1 (0 means no stripe run), and outlives
+ * stripe_run: the finish path uses it to defer the file extent to
+ * the next transaction when the joined transaction's commit did not
+ * retire this run's window.
*/
struct btrfs_open_stripe_run *stripe_run;
+ u64 stripe_seq;
/* compression algorithm */
int compress_type;
ret = false;
}
+ if (btrfs_raw_test_opt(*mount_opt, STRIPE_ALLOC)) {
+ if (!btrfs_raw_test_opt(*mount_opt, FREE_SPACE_TREE)) {
+ btrfs_err(info,
+ "stripe_alloc requires the free space tree (space_cache=v2)");
+ ret = false;
+ }
+ if (btrfs_is_zoned(info)) {
+ btrfs_err(info,
+ "stripe_alloc is not supported on zoned filesystems");
+ ret = false;
+ }
+ if (btrfs_fs_incompat(info, MIXED_GROUPS)) {
+ btrfs_err(info,
+ "stripe_alloc is not supported on mixed block groups");
+ ret = false;
+ }
+ }
+
if (btrfs_check_mountopts_zoned(info, mount_opt))
ret = false;
* their tails, drains (instantly, all IO reported done) and drops
* the block group's membership in fs_info->open_stripe_bgs.
*/
- btrfs_retire_open_stripes(fs_info);
+ btrfs_retire_open_stripes(fs_info, NULL);
if (!test_check_exists(cache, 3 * fsl + 2 * sectorsize,
fsl - 2 * sectorsize)) {
test_err("retired cow run tail missing from free space");
goto out;
}
btrfs_open_stripe_write_done(cache, 5 * fsl, sectorsize);
- btrfs_retire_open_stripes(fs_info);
+ btrfs_retire_open_stripes(fs_info, NULL);
/* Only partially filled stripes remain: nothing left to claim. */
ret = btrfs_alloc_from_open_stripe(cache, sectorsize,
atomic_set(&cur_trans->pending_ordered, 0);
init_waitqueue_head(&cur_trans->pending_wait);
atomic_set(&cur_trans->num_writers, 1);
+ cur_trans->stripe_retire_seq = 0;
extwriter_counter_init(cur_trans, type);
init_waitqueue_head(&cur_trans->writer_wait);
init_waitqueue_head(&cur_trans->commit_wait);
!TRANS_ABORTED(trans));
}
+/*
+ * Wait for the given transaction, if it is still the running one, to reach
+ * TRANS_STATE_UNBLOCKED (or abort). Used by ordered extent completion to
+ * defer a file extent whose stripe run window the committing transaction
+ * did not retire: joining again afterwards attaches to a later transaction.
+ */
+void btrfs_wait_transid_unblocked(struct btrfs_fs_info *fs_info, u64 transid)
+{
+ struct btrfs_transaction *cur_trans = NULL;
+
+ spin_lock(&fs_info->trans_lock);
+ if (fs_info->running_transaction &&
+ fs_info->running_transaction->transid == transid) {
+ cur_trans = fs_info->running_transaction;
+ refcount_inc(&cur_trans->use_count);
+ }
+ spin_unlock(&fs_info->trans_lock);
+ if (!cur_trans)
+ return;
+ wait_event(fs_info->transaction_wait,
+ cur_trans->state >= TRANS_STATE_UNBLOCKED ||
+ TRANS_ABORTED(cur_trans));
+ btrfs_put_transaction(cur_trans);
+}
+
/* wait for commit against the current transaction to become unblocked
* when this is done, it is safe to start a new transaction, but the current
* transaction might not be fully on disk.
wait_event(cur_trans->pending_wait,
atomic_read(&cur_trans->pending_ordered) == 0);
+ /*
+ * Retire this window's open stripe runs and wait for all data IO
+ * into their stripes to reach disk, closing the raid56 write hole
+ * for stripe-allocated block groups: after the drain, nothing will
+ * ever write those stripes again, so a crash after the superblock
+ * write cannot tear a stripe this transaction references.
+ *
+ * This must run while the transaction still accepts joins: tasks
+ * holding unreported stripe run bytes (a reserved extent whose bio
+ * has not completed) can block on a transaction join, e.g. for a
+ * chunk allocation while walking a delalloc range, and the drain
+ * waits for those bytes -- retiring after joins are blocked would
+ * deadlock. The window this opens -- an extent from a not-retired
+ * run completing fast and joining this transaction before
+ * TRANS_STATE_COMMIT_DOING -- is closed at the ordered extent
+ * finish: extents stamped with a stripe run sequence this
+ * transaction did not retire wait and join the next one. Cheap
+ * no-op when no stripe runs exist.
+ */
+ btrfs_retire_open_stripes(fs_info, cur_trans);
+
btrfs_scrub_pause(fs_info);
/*
* Ok now we need to make sure to block out any other joins while we
struct btrfs_transaction {
u64 transid;
+ /*
+ * The stripe retire sequence this transaction's commit bumped to, 0
+ * until its commit retires the open stripe runs. An ordered extent
+ * from a run with open_seq >= this value must not add its file
+ * extent to this transaction (the run was not retired and drained
+ * by it); see btrfs_retire_open_stripes(). Written and read under
+ * fs_info->open_stripe_lock.
+ */
+ u64 stripe_retire_seq;
/*
* total external writers(USERSPACE/START/ATTACH) in this
* transaction, it must be zero before the transaction is
struct btrfs_trans_handle *btrfs_attach_transaction_barrier(
struct btrfs_root *root);
int btrfs_wait_for_commit(struct btrfs_fs_info *fs_info, u64 transid);
+void btrfs_wait_transid_unblocked(struct btrfs_fs_info *fs_info, u64 transid);
void btrfs_add_dead_root(struct btrfs_root *root);
void btrfs_maybe_wake_unfinished_drop(struct btrfs_fs_info *fs_info);
projects / linux / commitdiff