*/
unsigned long *written;
u32 live_nbits; /* sectors @live covers; caps growth */
+ /* META_LOG: the owning subvolume's log_transid this run belongs to. */
+ u64 log_transid;
};
-static void btrfs_dump_open_meta_runs(struct btrfs_block_group *bg)
+static void btrfs_dump_open_stripe_runs(struct btrfs_block_group *bg)
{
struct btrfs_open_stripe_run *run;
unsigned long flags;
spin_lock_irqsave(&bg->stripe_run_lock, flags);
- list_for_each_entry(run, &bg->open_stripe_runs, list) {
- if (run->class != BTRFS_STRIPE_RUN_META)
- continue;
+ list_for_each_entry(run, &bg->open_stripe_runs, list)
btrfs_warn(bg->fs_info,
-" run %llu..%llu frontier %llu open %d inflight %llu transid %llu",
- run->start, run->end, run->offset, run->open,
- run->inflight_bytes, run->transid);
- }
+" run %llu..%llu class %d frontier %llu open %d inflight %llu transid %llu",
+ run->start, run->end, run->class, run->offset,
+ run->open, run->inflight_bytes, run->transid);
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
}
spin_lock_irqsave(&bg->stripe_run_lock, flags);
list_for_each_entry(run, &bg->open_stripe_runs, list) {
- if (run->class != BTRFS_STRIPE_RUN_META)
+ if (!btrfs_stripe_run_class_is_meta(run->class))
continue;
/*
* An open run here belongs to the running transaction: the
*/
static bool run_freed_on_drain(const struct btrfs_open_stripe_run *run)
{
- return run->class != BTRFS_STRIPE_RUN_META && !run->retained;
+ return !btrfs_stripe_run_class_is_meta(run->class) && !run->retained;
}
static void free_open_stripe_run(struct btrfs_block_group *bg,
return ret;
}
+/*
+ * Allocate a log tree block from the private run of this log commit.
+ *
+ * The run is keyed by @owner (the subvolume's root id) and @log_transid, so
+ * each log commit gets whole stripes of its own: its blocks are durable once
+ * fsync returns, and no later writer may share a stripe with them (see
+ * BTRFS_STRIPE_RUN_META_LOG). An open run of the same owner from an earlier
+ * log commit is closed, as a stale-transid metadata run is in
+ * btrfs_alloc_from_open_stripe(). Like the other private runs, it is never
+ * placed in a band slot.
+ *
+ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in this
+ * block group can satisfy the allocation, or -ENOMEM.
+ */
+int btrfs_alloc_from_log_stripe_run(struct btrfs_block_group *bg, u64 num_bytes,
+ u64 owner, u64 log_transid,
+ u64 *ret_offset, u64 *available)
+{
+ struct btrfs_fs_info *fs_info = bg->fs_info;
+ const u64 fsl = bg->full_stripe_len;
+ const enum btrfs_stripe_run_class class = BTRFS_STRIPE_RUN_META_LOG;
+ struct btrfs_open_stripe_run *new_run;
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ u64 tail_start = 0;
+ u64 tail_len = 0;
+ u64 start;
+ u64 len;
+ int ret;
+
+ ASSERT(num_bytes);
+ *available = 0;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (run->class != class || run->owner != owner || !run->open)
+ continue;
+ /*
+ * A previous log commit of the same subvolume. Its blocks are
+ * durable, so this one must not continue into them; close it
+ * and claim fresh stripes below. log_transid starts over with
+ * every transaction's log tree, so a run from an earlier
+ * transaction can carry the same log_transid: the transid
+ * tells them apart. Joining it would hand that transaction's
+ * commit, which retires the run and waits for its blocks, a
+ * block only this transaction will write.
+ */
+ if (run->log_transid != log_transid ||
+ run->transid != btrfs_get_fs_generation(fs_info)) {
+ tail_len = close_open_stripe_run(bg, run, &tail_start);
+ break;
+ }
+ if (num_bytes <= run->end - run->offset) {
+ *ret_offset = run->offset;
+ run_live_set(run, run->offset, num_bytes, true);
+ run->offset += num_bytes;
+ run->inflight_bytes += num_bytes;
+ bg->stripe_open_remainder -= num_bytes;
+ if (run->offset == run->end)
+ run->open = false; /* full: closed, no tail */
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ stripe_open_remainder_sync(bg);
+ return 0;
+ }
+ /* Too small for this block: replace it. */
+ tail_len = close_open_stripe_run(bg, run, &tail_start);
+ break;
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ if (tail_len) {
+ btrfs_add_free_space(bg, tail_start, tail_len);
+ stripe_open_remainder_sync(bg);
+ }
+
+ /* Zeroed: the error paths below free ->live and must find NULL. */
+ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
+ if (!new_run)
+ return -ENOMEM;
+
+ ret = btrfs_claim_free_stripe_run(bg,
+ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
+ &start, &len);
+ if (ret)
+ goto out;
+ if (len < num_bytes) {
+ btrfs_add_free_space(bg, start, len);
+ *available = max(*available, len);
+ ret = -ENOSPC;
+ goto out;
+ }
+
+ /* Sleeps, so it has to happen before the locks below. */
+ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
+ new_run->written = new_run->live_nbits ?
+ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
+
+ spin_lock(&fs_info->open_stripe_lock);
+ if (list_empty(&bg->open_stripe_bg_list)) {
+ btrfs_get_block_group(bg);
+ list_add_tail(&bg->open_stripe_bg_list,
+ &fs_info->open_stripe_bgs);
+ }
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ if (bg->ro) {
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ spin_unlock(&fs_info->open_stripe_lock);
+ btrfs_add_free_space(bg, start, len);
+ *available = 0;
+ ret = -ENOSPC;
+ goto out;
+ }
+ new_run->bg = bg;
+ new_run->class = class;
+ new_run->persistent = false;
+ new_run->root = owner;
+ new_run->owner = owner;
+ new_run->log_transid = log_transid;
+ new_run->transid = btrfs_get_fs_generation(fs_info);
+ new_run->start = start;
+ new_run->end = start + len;
+ new_run->offset = start + num_bytes;
+ new_run->inflight_bytes = num_bytes;
+ new_run->open_seq = fs_info->stripe_retire_seq;
+ new_run->open = (new_run->offset != new_run->end);
+ run_live_set(new_run, start, num_bytes, true);
+ list_add_tail(&new_run->list, &bg->open_stripe_runs);
+ bg->stripe_open_remainder += new_run->end - new_run->offset;
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ spin_unlock(&fs_info->open_stripe_lock);
+
+ atomic64_inc(&fs_info->stripe_run_stats.log_run_opened);
+ stripe_open_remainder_sync(bg);
+ *ret_offset = start;
+ new_run = NULL;
+ ret = 0;
+out:
+ if (new_run) {
+ bitmap_free(new_run->live);
+ bitmap_free(new_run->written);
+ }
+ kfree(new_run);
+ return ret;
+}
+
/*
* The owner a private run is found by: the inode, or with
* stripe_log_data_shared a LOG-class run shared by every inode of the
* commit and their tree blocks are not written until later;
* btrfs_retire_meta_stripes() closes them once they are.
*/
- if (run->class == BTRFS_STRIPE_RUN_META && seq != U64_MAX)
+ if (btrfs_stripe_run_class_is_meta(run->class) && seq != U64_MAX)
continue;
for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
seq != U64_MAX)
continue;
/* Metadata drains after tree writeback, not here. */
- if (run->class == BTRFS_STRIPE_RUN_META && seq != U64_MAX)
+ if (btrfs_stripe_run_class_is_meta(run->class) && seq != U64_MAX)
continue;
if (run->open_seq < seq) {
ret = false;
spin_unlock(&fs_info->open_stripe_lock);
}
+/* Does @bg have metadata stripe runs? */
+static bool bg_has_meta_runs(struct btrfs_block_group *bg)
+{
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ bool ret = false;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (btrfs_stripe_run_class_is_meta(run->class)) {
+ ret = true;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ return ret;
+}
+
/*
* Close and drain all of a block group's stripe runs. The caller must
* prevent new allocations first (e.g. the block group is read-only),
void btrfs_retire_block_group_stripes(struct btrfs_block_group *bg)
{
struct btrfs_fs_info *fs_info = bg->fs_info;
+ bool warned = false;
/* Nothing may outlive this, relocation's retained runs included. */
release_retained_stripe_runs(bg);
close_block_group_stripe_runs(bg, U64_MAX);
/* Parked partial-stripe rbios hold the very bios the wait drains. */
btrfs_flush_parked_rbios(fs_info, bg->start, bg->length, false);
- wait_var_event(&bg->open_stripe_runs,
- bg_open_stripes_settled(bg, U64_MAX) ||
- BTRFS_FS_ERROR(fs_info));
+ /*
+ * Metadata runs drain when their tree blocks are written and are
+ * freed by btrfs_retire_meta_stripes(), both in a commit. Commit now
+ * instead of waiting for the next one; no caller holds a transaction
+ * handle.
+ */
+ if (bg_has_meta_runs(bg))
+ btrfs_commit_current_transaction(fs_info->tree_root);
+ /*
+ * No timeout: a run that does not drain has an allocation that was
+ * never reported written or abandoned, an accounting bug to fix, not
+ * to wait out. Say which runs once, and keep waiting.
+ */
+ while (!wait_var_event_timeout(&bg->open_stripe_runs,
+ bg_open_stripes_settled(bg, U64_MAX) ||
+ BTRFS_FS_ERROR(fs_info), 60 * HZ)) {
+ if (warned)
+ continue;
+ btrfs_warn(fs_info,
+"block group %llu read-only transition waiting for stripe runs that do not drain",
+ bg->start);
+ btrfs_dump_open_stripe_runs(bg);
+ warned = true;
+ }
if (BTRFS_FS_ERROR(fs_info) && !bg_open_stripes_settled(bg, U64_MAX))
btrfs_warn(fs_info,
"block group %llu read-only with stripe runs still in flight after a filesystem error; not waiting for them",
while (!list_empty(&retire_list)) {
struct btrfs_open_stripe_run *run;
bool again = true;
+ bool warned = false;
bg = list_first_entry(&retire_list, struct btrfs_block_group,
open_stripe_retire_list);
again = false;
spin_lock_irqsave(&bg->stripe_run_lock, flags);
list_for_each_entry(run, &bg->open_stripe_runs, list) {
- if (run->class != BTRFS_STRIPE_RUN_META ||
+ if (!btrfs_stripe_run_class_is_meta(run->class) ||
!run->open)
continue;
/*
/*
* Every tree block is on disk, so end_bbio_meta_write() has
- * reported all of them and this normally does not wait. The
- * timeout guards against a path that frees a tree block
- * without reporting it to its run: warn and keep the run,
- * which costs space, rather than hang the commit.
+ * reported all of them and this normally does not wait. A run
+ * that does not drain has a tree block that was freed without
+ * reporting to its run, an accounting bug: say which runs
+ * once, and keep waiting. Only a filesystem error ends the
+ * wait early.
*/
- if (!wait_var_event_timeout(&bg->open_stripe_runs,
- !bg_has_undrained_meta_runs(bg, transid),
- 60 * HZ)) {
+ while (!wait_var_event_timeout(&bg->open_stripe_runs,
+ !bg_has_undrained_meta_runs(bg, transid) ||
+ BTRFS_FS_ERROR(fs_info), 60 * HZ)) {
+ if (warned)
+ continue;
btrfs_warn(fs_info,
-"metadata stripe runs in block group %llu did not drain; a tree block was freed without reporting to its run",
+"metadata stripe runs in block group %llu do not drain; a tree block was freed without reporting to its run",
bg->start);
- btrfs_dump_open_meta_runs(bg);
+ btrfs_dump_open_stripe_runs(bg);
+ warned = true;
}
/* Drained and no longer needed for padding or merging. */
struct btrfs_open_stripe_run *next = NULL;
list_for_each_entry(run, &bg->open_stripe_runs, list) {
- if (run->class == BTRFS_STRIPE_RUN_META &&
+ if (btrfs_stripe_run_class_is_meta(run->class) &&
open_stripe_run_drained(run)) {
next = run;
break;
struct btrfs_fs_info *fs_info = to_fs_info(kobj);
/* Indexed by enum btrfs_stripe_run_class. */
static const char * const class_names[] = {
- "cow", "reloc", "log", "nocow", "meta",
+ "cow", "reloc", "log", "nocow", "meta", "meta_log",
};
int live_by_class[BTRFS_STRIPE_RUN_NR_CLASSES];
int open_runs[BTRFS_STRIPE_RUN_NR_CLASSES];
"compact_runs %lld\n"
"compact_fallbacks %lld\n"
"compact_stripes %lld\n"
- "compact_bytes %lld\n",
+ "compact_bytes %lld\n"
+ "log_run_opened %lld\n",
atomic64_read(&fs_info->stripe_run_stats.nocow_run_opened),
atomic64_read(&fs_info->stripe_run_stats.claim_refused_live_run),
atomic64_read(&fs_info->stripe_run_stats.strand_scan_bytes),
atomic64_read(&fs_info->stripe_run_stats.compact_runs),
atomic64_read(&fs_info->stripe_run_stats.compact_fallbacks),
atomic64_read(&fs_info->stripe_run_stats.compact_stripes),
- atomic64_read(&fs_info->stripe_run_stats.compact_bytes));
+ atomic64_read(&fs_info->stripe_run_stats.compact_bytes),
+ atomic64_read(&fs_info->stripe_run_stats.log_run_opened));
}
BTRFS_ATTR(, stripe_run_stats, btrfs_stripe_run_stats_show);
BTRFS_PARK_TIMEOUT_ATTR(stripe_park_timeout_ms, stripe_park_timeout_ms);
+/*
+ * Whether each log commit gets its own private metadata stripe run. 0 puts
+ * log tree blocks back in the transaction's shared metadata runs, which
+ * reopens the log-tree fsync write hole.
+ */
+static ssize_t btrfs_stripe_log_run_show(struct kobject *kobj,
+ struct kobj_attribute *a, char *buf)
+{
+ struct btrfs_fs_info *fs_info = to_fs_info(kobj);
+
+ return sysfs_emit(buf, "%d\n", fs_info->stripe_log_run ? 1 : 0);
+}
+
+static ssize_t btrfs_stripe_log_run_store(struct kobject *kobj,
+ struct kobj_attribute *a,
+ const char *buf, size_t len)
+{
+ struct btrfs_fs_info *fs_info = to_fs_info(kobj);
+ bool val;
+ int ret;
+
+ ret = kstrtobool(buf, &val);
+ if (ret)
+ return ret;
+ fs_info->stripe_log_run = val;
+ return len;
+}
+BTRFS_ATTR_RW(, stripe_log_run, btrfs_stripe_log_run_show,
+ btrfs_stripe_log_run_store);
+
static ssize_t btrfs_stripe_log_data_shared_show(struct kobject *kobj,
struct kobj_attribute *a, char *buf)
{
BTRFS_ATTR_PTR(, stripe_park_stats),
BTRFS_ATTR_PTR(, stripe_run_stats),
BTRFS_ATTR_PTR(, stripe_park_timeout_ms),
+ BTRFS_ATTR_PTR(, stripe_log_run),
BTRFS_ATTR_PTR(, stripe_log_data_shared),
BTRFS_ATTR_PTR(, stripe_park_sync_timeout_ms),
BTRFS_ATTR_PTR(, stripe_park_congestion),