u64 offset; /* next unallocated byte */
u64 inflight_bytes; /* reserved bytes with data IO pending */
u64 open_seq; /* fs_info->stripe_retire_seq at open */
+ u64 transid; /* fs_info->generation at open */
u64 owner; /* LOG class: btrfs_ino of the owner */
enum btrfs_stripe_run_class class;
bool open; /* accepting allocations */
u32 live_nbits; /* sectors @live covers; caps growth */
};
+static void btrfs_dump_open_meta_runs(struct btrfs_block_group *bg)
+{
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (run->class != BTRFS_STRIPE_RUN_META)
+ continue;
+ btrfs_warn(bg->fs_info,
+" run %llu..%llu frontier %llu open %d inflight %llu transid %llu",
+ run->start, run->end, run->offset, run->open,
+ run->inflight_bytes, run->transid);
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+}
+
+static bool bg_has_undrained_meta_runs(struct btrfs_block_group *bg,
+ u64 transid)
+{
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ bool ret = false;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (run->class != BTRFS_STRIPE_RUN_META)
+ continue;
+ /*
+ * An open run here belongs to the running transaction: the
+ * retirer closed every run it is responsible for before its
+ * wait, so this one was opened while it waited, and drains
+ * when that transaction's own commit writes its tree blocks.
+ * Waiting on it would be waiting for writes this commit never
+ * issued.
+ */
+ if (run->open)
+ continue;
+ /*
+ * Likewise a run the running transaction opened and a band
+ * collision or eviction closed early: its blocks are written,
+ * and its stripes settled, by that transaction's own commit.
+ */
+ if (run->transid > transid)
+ continue;
+ if (run->inflight_bytes) {
+ ret = true;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ return ret;
+}
+
static bool open_stripe_run_drained(const struct btrfs_open_stripe_run *run)
{
return !run->open && run->inflight_bytes == 0;
bitmap_clear(run->live, first, nr);
}
-/* Free a drained run. Caller holds bg->stripe_run_lock. */
+/*
+ * A drained run may still be needed.
+ *
+ * Metadata writeback issues one rbio per contiguous run of tree blocks, so
+ * a stripe's columns can be submitted separately. If the run were freed as
+ * soon as its counter hit zero -- which happens when the first column's
+ * completions arrive -- the second column would find no run covering its
+ * stripe, and with no run there is nothing to pad against and nothing to
+ * merge with. Metadata runs are therefore freed by
+ * btrfs_retire_meta_stripes(), which runs once every tree block is on disk.
+ */
+static bool run_freed_on_drain(const struct btrfs_open_stripe_run *run)
+{
+ return run->class != BTRFS_STRIPE_RUN_META;
+}
+
static void free_open_stripe_run(struct btrfs_block_group *bg,
struct btrfs_open_stripe_run *run)
{
wake_up_var(&bg->open_stripe_runs);
}
+/*
+ * Report a run's transition to drained. Freeing wakes the retire waiters as
+ * a side effect of free_open_stripe_run(); a metadata run instead stays
+ * behind for padding and merging until btrfs_retire_meta_stripes() collects
+ * it, but that same retirer may already be asleep waiting for exactly this
+ * drain -- completion reports normally all precede its wait, but a report
+ * can arrive outside the writeback window (a tree block freed before it was
+ * written, e.g. an unwritten log tree block). Wake it, or it sleeps out
+ * its whole timeout.
+ */
+static void open_stripe_run_drain_reported(struct btrfs_block_group *bg,
+ struct btrfs_open_stripe_run *run)
+{
+ if (!open_stripe_run_drained(run))
+ return;
+ if (run_freed_on_drain(run))
+ free_open_stripe_run(bg, run);
+ else
+ wake_up_var(&bg->open_stripe_runs);
+}
+
/*
* Close a run: no further allocation may join it, and its unallocated tail
* must be returned to the free space cache by the caller (after dropping the
u64 *ret_offset, u64 *available)
{
struct btrfs_fs_info *fs_info = bg->fs_info;
+ const u64 transid = btrfs_get_fs_generation(fs_info);
+ u64 evict_start[BTRFS_STRIPE_RUN_NR_BANDS];
+ u64 evict_len[BTRFS_STRIPE_RUN_NR_BANDS];
+ int nr_evict = 0;
const u64 fsl = bg->full_stripe_len;
struct btrfs_open_stripe_run *new_run;
struct btrfs_open_stripe_run *run;
if (!r)
continue;
+ /*
+ * Metadata runs are closed after the next transaction is
+ * already running (btrfs_retire_meta_stripes()), so a slot can
+ * still hold the previous transaction's run. Joining it would
+ * put two transactions' tree blocks in the same stripes, and
+ * the later one's writeback would read-modify-write parity
+ * over the earlier one's committed blocks: the write hole.
+ * Close it; its tail returns to the free space cache below.
+ */
+ if (class == BTRFS_STRIPE_RUN_META && r->transid != transid) {
+ bg->open_stripe[class][band] = NULL;
+ evict_len[nr_evict] =
+ close_open_stripe_run(bg, r,
+ &evict_start[nr_evict]);
+ if (evict_len[nr_evict])
+ nr_evict++;
+ continue;
+ }
rem = r->end - r->offset;
if (rem > *available)
*available = rem;
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
if (tail_len)
btrfs_add_free_space(bg, tail_start, tail_len);
+ for (int i = 0; i < nr_evict; i++)
+ btrfs_add_free_space(bg, evict_start[i], evict_len[i]);
stripe_open_remainder_sync(bg);
return 0;
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ for (int i = 0; i < nr_evict; i++)
+ btrfs_add_free_space(bg, evict_start[i], evict_len[i]);
+ if (nr_evict)
+ stripe_open_remainder_sync(bg);
/*
* Zeroed: the error paths below reach out: before the liveness map is
if (!r || r->end != start)
continue;
+ /* Never grow another transaction's run (see the scan above). */
+ if (class == BTRFS_STRIPE_RUN_META && r->transid != transid)
+ continue;
/*
* Only grow as far as the run's liveness map reaches; it
* cannot be reallocated here (see alloc_run_live_map()).
new_run->bg = bg;
new_run->class = class;
new_run->owner = 0;
+ new_run->transid = transid;
new_run->start = start;
new_run->end = start + len;
new_run->offset = start + num_bytes;
ASSERT(run->inflight_bytes >= num_bytes);
run_live_set(run, start, num_bytes, false);
run->inflight_bytes -= num_bytes;
- if (open_stripe_run_drained(run))
- free_open_stripe_run(bg, run);
+ open_stripe_run_drain_reported(bg, run);
break;
}
}
ASSERT(start + num_bytes <= run->offset);
ASSERT(run->inflight_bytes >= num_bytes);
run->inflight_bytes -= num_bytes;
- if (open_stripe_run_drained(run))
- free_open_stripe_run(bg, run);
+ open_stripe_run_drain_reported(bg, run);
break;
}
}
spin_lock_irqsave(&bg->stripe_run_lock, flags);
ASSERT(run->inflight_bytes >= num_bytes);
run->inflight_bytes -= num_bytes;
- if (open_stripe_run_drained(run))
- free_open_stripe_run(bg, run);
+ open_stripe_run_drain_reported(bg, run);
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
}
*/
if (run->class == BTRFS_STRIPE_RUN_NOCOW && seq != U64_MAX)
continue;
+ /*
+ * Metadata runs are still being filled at this point in the
+ * commit and their tree blocks are not written until later;
+ * btrfs_retire_meta_stripes() closes them once they are.
+ */
+ if (run->class == BTRFS_STRIPE_RUN_META && seq != U64_MAX)
+ continue;
for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
if (bg->open_stripe[class][band] == run)
/* Persistent nocow runs are not part of any commit window. */
if (run->class == BTRFS_STRIPE_RUN_NOCOW && seq != U64_MAX)
continue;
+ /* Metadata drains after tree writeback, not here. */
+ if (run->class == BTRFS_STRIPE_RUN_META && seq != U64_MAX)
+ continue;
if (run->open_seq < seq) {
ret = false;
break;
* Calls are serialized by the transaction commit; @trans may be NULL for
* the final cleanup call.
*/
+/*
+ * Close and drain the metadata stripe runs, once every tree block this
+ * transaction allocated has reached the disk.
+ *
+ * This is the second half of a two-phase closure. The first half happens
+ * implicitly: btrfs_retire_open_stripes() runs earlier in the commit and
+ * skips these runs, so they stay open for allocation right through the
+ * commit-time tree building that follows it. Here the allocation side is
+ * finished and the writes are complete, so the runs can be closed and their
+ * unused tails returned.
+ *
+ * Ordering matters in both directions. Closing earlier would strand tree
+ * blocks the commit has not allocated yet; draining earlier would wait for
+ * writes the commit has not issued yet, which is precisely the deadlock
+ * mixed block groups used to hit.
+ */
+void btrfs_retire_meta_stripes(struct btrfs_fs_info *fs_info, u64 transid)
+{
+ struct btrfs_block_group *bg;
+ LIST_HEAD(retire_list);
+ unsigned long flags;
+
+ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+ return;
+
+ spin_lock(&fs_info->open_stripe_lock);
+ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
+ list_add_tail(&bg->open_stripe_retire_list, &retire_list);
+ spin_unlock(&fs_info->open_stripe_lock);
+
+ while (!list_empty(&retire_list)) {
+ struct btrfs_open_stripe_run *run;
+ bool again = true;
+
+ bg = list_first_entry(&retire_list, struct btrfs_block_group,
+ open_stripe_retire_list);
+ list_del_init(&bg->open_stripe_retire_list);
+ if (!(bg->flags & BTRFS_BLOCK_GROUP_METADATA))
+ continue;
+
+ while (again) {
+ u64 tail_start;
+ u64 tail_len = 0;
+
+ again = false;
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (run->class != BTRFS_STRIPE_RUN_META ||
+ !run->open)
+ continue;
+ /*
+ * The running transaction's runs are not ours
+ * to close: their tree blocks are written, and
+ * their stripes settled, by that transaction's
+ * own commit.
+ */
+ if (run->transid > transid)
+ continue;
+ for (int band = 0;
+ band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+ if (bg->open_stripe[BTRFS_STRIPE_RUN_META][band] == run)
+ bg->open_stripe[BTRFS_STRIPE_RUN_META][band] = NULL;
+ tail_len = close_open_stripe_run(bg, run, &tail_start);
+ again = true;
+ break;
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ if (tail_len)
+ btrfs_add_free_space(bg, tail_start, tail_len);
+ }
+
+ /*
+ * Every tree block is on disk, so end_bbio_meta_write() has
+ * reported all of them and this settles without waiting.
+ *
+ * Bounded rather than indefinite on purpose. A run drains
+ * only when every byte allocated from it has been reported,
+ * by writeback or by whatever freed the block before it was
+ * written; a path that frees a tree block without reporting
+ * would otherwise wedge the commit with no clue as to which
+ * one. Time out, say so, and carry on: the stripe is then
+ * left open, which costs space, where hanging costs the
+ * filesystem.
+ */
+ if (!wait_var_event_timeout(&bg->open_stripe_runs,
+ !bg_has_undrained_meta_runs(bg, transid),
+ 60 * HZ)) {
+ btrfs_warn(fs_info,
+"metadata stripe runs in block group %llu did not drain; a tree block was freed without reporting to its run",
+ bg->start);
+ btrfs_dump_open_meta_runs(bg);
+ }
+
+ /* Drained and no longer needed for padding or merging. */
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ while (true) {
+ struct btrfs_open_stripe_run *next = NULL;
+
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (run->class == BTRFS_STRIPE_RUN_META &&
+ open_stripe_run_drained(run)) {
+ next = run;
+ break;
+ }
+ }
+ if (!next)
+ break;
+ free_open_stripe_run(bg, next);
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ }
+}
+
+/*
+ * Should this tree block be left dirty for the transaction commit to write?
+ *
+ * A stripe filled by one transaction is written in one piece only if all of
+ * its tree blocks go out in the same writeback pass. The opportunistic
+ * passes -- memory reclaim, the periodic writeback timer -- do not know
+ * about stripes and will happily write three tree blocks now and the other
+ * five later, which turns a full-stripe write into two read-modify-writes.
+ *
+ * Zoned mode has the same problem for a different reason (a tree block that
+ * is not at its zone's write pointer cannot be written yet) and solves it
+ * the same way: btrfs_check_meta_write_pointer() declines the block and the
+ * writeback loop leaves it dirty. This declines on stripe grounds instead.
+ *
+ * WB_SYNC_ALL is never deferred. That is what keeps this from deadlocking:
+ * a reclaimer that must have these pages written waits on WB_SYNC_ALL, which
+ * always proceeds, so dirty metadata pinned in an incomplete stripe can
+ * always be flushed by the caller who needs the memory. Only the
+ * best-effort passes are turned away, and turning one away asks for a commit
+ * rather than leaving the pages for the periodic writeback to find again.
+ */
+bool btrfs_defer_stripe_meta_write(struct btrfs_fs_info *fs_info,
+ struct extent_buffer *eb,
+ const struct writeback_control *wbc)
+{
+ struct btrfs_block_group *bg;
+ bool defer;
+
+ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+ return false;
+ if (wbc->sync_mode == WB_SYNC_ALL)
+ return false;
+
+ bg = btrfs_lookup_block_group(fs_info, eb->start);
+ if (!bg)
+ return false;
+ defer = btrfs_is_stripe_meta_bg(bg);
+ btrfs_put_block_group(bg);
+
+ if (defer) {
+ set_bit(BTRFS_FS_NEED_TRANS_COMMIT, &fs_info->flags);
+ wake_up_process(fs_info->transaction_kthread);
+ }
+ return defer;
+}
+
void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info,
struct btrfs_transaction *trans)
{
bg = btrfs_lookup_block_group(fs_info, full_stripe_start);
if (!bg)
return;
- if (!btrfs_is_stripe_alloc_bg(bg) ||
+ /*
+ * Metadata is checked on the same terms once stripe_meta places it:
+ * its runs stay open across the commit's tree building but are closed
+ * and drained before the commit ends, so a tree block write outside
+ * every live run means a stripe was revisited after the transaction
+ * that filled it finished -- the same violation as for data.
+ */
+ if ((!btrfs_is_stripe_alloc_bg(bg) && !btrfs_is_stripe_meta_bg(bg)) ||
test_bit(BLOCK_GROUP_FLAG_STRIPE_RELOC_USED, &bg->runtime_flags) ||
test_bit(BLOCK_GROUP_FLAG_STRIPE_NOCOW_USED, &bg->runtime_flags))
goto out;
*/
static void warn_stripe_alloc_uncovered(struct btrfs_fs_info *info)
{
- const u64 meta_bits = info->avail_metadata_alloc_bits |
- info->avail_system_alloc_bits;
-
if (!btrfs_test_opt(info, STRIPE_ALLOC))
return;
- if (meta_bits & BTRFS_BLOCK_GROUP_RAID56_MASK)
+ /*
+ * raid56 metadata is covered now that the policy follows the chunk
+ * profile. System chunks are not: btrfs_is_stripe_meta_bg() tests
+ * the METADATA flag, and a system chunk does not carry it, so a
+ * raid56 system chunk keeps the legacy read-modify-write and its
+ * write hole. They are small and rewritten rarely, but say so
+ * rather than let the earlier "covered" message imply otherwise.
+ */
+ if (info->avail_system_alloc_bits & BTRFS_BLOCK_GROUP_RAID56_MASK)
btrfs_warn(info,
-"stripe_alloc does not cover raid56 metadata: the write hole remains for metadata, consider converting it to raid1c3/raid1c4");
+"stripe_alloc does not cover raid56 system chunks: the write hole remains for them, consider converting to raid1c3/raid1c4");
}
int btrfs_read_block_groups(struct btrfs_fs_info *info)
if (btrfs_header_generation(buf) != trans->transid)
return 0;
+ /*
+ * Last ref to a tree block this transaction allocated and never
+ * wrote: no writeback completion will ever report it to its stripe
+ * run. Report it here, before the free paths diverge. When the
+ * delayed ref head was already processed (truncate and friends run
+ * delayed refs mid-transaction for throttling) check_ref_cleanup()
+ * below returns 0 and the free goes through __btrfs_free_extent()
+ * and the pin/unpin machinery, none of which know about the run --
+ * the run's inflight count would never drain.
+ */
+ if (!btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
+ bg = btrfs_lookup_block_group(fs_info, buf->start);
+ if (btrfs_is_stripe_meta_bg(bg))
+ btrfs_open_stripe_write_abandoned(bg, buf->start,
+ buf->len);
+ btrfs_put_block_group(bg);
+ }
+
if (root_id != BTRFS_TREE_LOG_OBJECTID) {
ret = check_ref_cleanup(trans, buf->start);
if (!ret)
(bg->flags & BTRFS_BLOCK_GROUP_RAID56_MASK);
}
+/*
+ * Should this block group allocate tree blocks by whole stripes?
+ *
+ * The same rule data follows: only fully free stripes are claimed, and the
+ * partial stripe left at the end of a commit is padded out rather than
+ * revisited, so no tree block write ever lands in a stripe holding earlier
+ * committed data. Not separately gated: the chunk profile already decides
+ * it. raid56 metadata has the same write hole as raid56 data and gets the
+ * same protection at the same time; metadata on any other profile has no
+ * write hole to close, and the RAID56_MASK test below means the policy
+ * simply does not apply to it. The costs metadata brings -- a stranded
+ * stripe tail per commit, and an allocation failure that aborts the
+ * transaction rather than failing one write -- are the price of covering
+ * raid56 metadata at all, and a filesystem that does not want them wants
+ * raid1c3/raid1c4 metadata instead, which is the better answer anyway.
+ */
+bool btrfs_is_stripe_meta_bg(const struct btrfs_block_group *bg)
+{
+ return btrfs_test_opt(bg->fs_info, STRIPE_ALLOC) &&
+ (bg->flags & BTRFS_BLOCK_GROUP_METADATA) &&
+ !(bg->flags & BTRFS_BLOCK_GROUP_DATA) &&
+ (bg->flags & BTRFS_BLOCK_GROUP_RAID56_MASK);
+}
+
static int do_allocation_stripe(struct btrfs_block_group *block_group,
struct find_free_extent_ctl *ffe_ctl,
struct btrfs_block_group **bg_ret)
bool skip = false;
int ret;
+ /*
+ * Tree blocks take the same path as data -- whole free stripes, a
+ * frontier that padding can complete -- in their own class, and none
+ * of the steering below applies to them: there is no inode to steer
+ * by, metadata is always COW, and relocation moves tree blocks
+ * through the normal metadata allocator.
+ */
+ if (btrfs_is_stripe_meta_bg(block_group)) {
+ ret = btrfs_alloc_from_open_stripe(block_group,
+ ffe_ctl->num_bytes,
+ BTRFS_STRIPE_RUN_META,
+ &offset, &available);
+ if (ret == -ENOMEM)
+ return ret;
+ if (ret) {
+ if (available > ffe_ctl->max_extent_size)
+ ffe_ctl->max_extent_size = available;
+ return 1;
+ }
+ ffe_ctl->found_offset = offset;
+ ffe_ctl->search_start = offset;
+ return 0;
+ }
+
/*
* Soft dedication of one block group to relocation, mirroring the
* zoned data_reloc_bg rule: it keeps relocation's prealloc-then-
static void undo_stripe_allocation(struct btrfs_block_group *block_group,
struct find_free_extent_ctl *ffe_ctl)
{
- if (btrfs_is_stripe_alloc_bg(block_group))
+ if (btrfs_is_stripe_alloc_bg(block_group) ||
+ btrfs_is_stripe_meta_bg(block_group))
btrfs_open_stripe_write_abandoned(block_group,
ffe_ctl->found_offset,
ffe_ctl->num_bytes);
{
switch (ffe_ctl->policy) {
case BTRFS_EXTENT_ALLOC_CLUSTERED:
- if (btrfs_is_stripe_alloc_bg(block_group))
+ if (btrfs_is_stripe_alloc_bg(block_group) ||
+ btrfs_is_stripe_meta_bg(block_group))
return do_allocation_stripe(block_group, ffe_ctl,
bg_ret);
return do_allocation_clustered(block_group, ffe_ctl, bg_ret);
projects / linux / commitdiff