if (list_empty_careful(&fs_info->open_stripe_bgs)) {
atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
+ if (info)
+ info->norun = BTRFS_STRIPE_NORUN_NO_BGS;
return false;
}
bg = btrfs_lookup_block_group(fs_info, stripe_start);
if (!bg) {
atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
+ if (info)
+ info->norun = BTRFS_STRIPE_NORUN_NO_BG;
return false;
}
+ if (info)
+ info->norun = BTRFS_STRIPE_NORUN_NO_COVER;
spin_lock_irqsave(&bg->stripe_run_lock, flags);
list_for_each_entry(run, &bg->open_stripe_runs, list) {
info->run_end = run->end;
info->run_offset = run->offset;
info->inflight_bytes = run->inflight_bytes;
+ info->norun = BTRFS_STRIPE_NORUN_NO_LIVE;
}
if (!run->live || !run->live_nbits)
break;
* needs to pad. An open run's end is stripe aligned, so
* overhang there means a stripe the run does not own.
*/
- if (stripe_start + stripe_len > run->end && run->open)
+ if (stripe_start + stripe_len > run->end && run->open) {
+ if (info)
+ info->norun = BTRFS_STRIPE_NORUN_OVERHANG;
break;
+ }
first = (stripe_start - run->start) >> bits;
/*
* which sectors are dead, and padding acts on that by
* overwriting them.
*/
- if (first + nsectors > run->live_nbits)
+ if (first + nsectors > run->live_nbits) {
+ if (info)
+ info->norun = BTRFS_STRIPE_NORUN_PAST_MAP;
break;
+ }
bitmap_zero(dead, nsectors);
if (unwritten)
bitmap_zero(unwritten, nsectors);
BTRFS_STRIPE_PAD_OK = 0,
BTRFS_STRIPE_PAD_NO_RUN, /* no open run covers the stripe */
BTRFS_STRIPE_PAD_UNCOVERED, /* a sector below the frontier is not covered */
+ BTRFS_STRIPE_PAD_NR_REASONS,
+};
+
+/*
+ * Which exit of btrfs_stripe_run_pad_mask() gave BTRFS_STRIPE_PAD_NO_RUN: a
+ * catch-all for every refusal, including "a run covers the stripe but its
+ * map cannot answer for it".
+ */
+enum btrfs_stripe_pad_norun {
+ BTRFS_STRIPE_NORUN_NO_BGS = 0, /* no block group has open runs */
+ BTRFS_STRIPE_NORUN_NO_BG, /* block group lookup failed */
+ BTRFS_STRIPE_NORUN_NO_COVER, /* no run's span covers the stripe */
+ BTRFS_STRIPE_NORUN_NO_LIVE, /* covering run has no live map */
+ BTRFS_STRIPE_NORUN_OVERHANG, /* open run's end cuts the stripe */
+ BTRFS_STRIPE_NORUN_PAST_MAP, /* stripe runs past the live map */
+ BTRFS_STRIPE_NORUN_NR,
};
struct btrfs_stripe_pad_info {
u64 run_offset; /* the allocation frontier */
u64 inflight_bytes;
u8 reason;
+ u8 norun; /* enum btrfs_stripe_pad_norun */
};
u64 btrfs_stripe_run_alloc_ceiling(struct btrfs_fs_info *fs_info,
* stripe.
*/
atomic64_t pad_decline_uncovered;
+ /*
+ * data_rmw split by why padding declined for the rbio that
+ * did it: 0..2 are enum btrfs_stripe_pad_reason (0: the pad
+ * oracle agreed but the data page allocation failed), 3: not
+ * tried (stripe_alloc off, or no memory for the bitmap).
+ * data_rmw_norun splits the NO_RUN entry by enum
+ * btrfs_stripe_pad_norun; norun_ro counts those in a
+ * read-only (relocating) group. reloc_writer counts data RMWs
+ * written by the data reloc inode outside the open run's
+ * frontier, disjoint from data_rmw_by[UNCOVERED].
+ */
+ atomic64_t data_rmw_by[4];
+ atomic64_t data_rmw_norun[6];
+ atomic64_t data_rmw_norun_ro;
+ atomic64_t data_rmw_reloc_writer;
atomic64_t meta_rmw;
atomic64_t congestion_short;
/* Stripes pinned in the rbio cache (rbio_needs_pin()). */
#include "file-item.h"
#include "btrfs_inode.h"
#include "block-group.h"
+#include "bio.h"
/* set when additional merges to this rbio are not allowed */
#define RBIO_RMW_LOCKED_BIT 1
/* Set at park time and never cleared: this rbio once parked (stats). */
#define RBIO_WAS_PARKED_BIT 7
+/* rbio->pad_reason when rmw_try_pad_full() did not consult the pad oracle. */
+#define RBIO_PAD_NOT_TRIED 0xff
+
/*
* Set when a parked rbio carries (or merged) a REQ_SYNC bio: a waiter
* is blocked on it, so deadlines stay authoritative and neither the
}
}
+static void rbio_record_pad(struct btrfs_raid_bio *rbio,
+ const struct btrfs_stripe_pad_info *info)
+{
+ rbio->pad_reason = info->reason;
+ rbio->pad_norun = info->norun;
+}
+
+static_assert(BTRFS_STRIPE_PAD_NR_REASONS + 1 ==
+ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_park_stats.data_rmw_by));
+static_assert(BTRFS_STRIPE_NORUN_NR ==
+ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_park_stats.data_rmw_norun));
+
+/*
+ * Attribute a read-modify-write to why padding declined for this rbio. With
+ * allow_rmw empty, a data RMW in a stripe_alloc group is expected in two
+ * places only: at the open run's own frontier (UNCOVERED: the sectors it
+ * reads are the same run's, in the same transaction, not yet durable), and
+ * from relocation writing into its own preallocated extents, which nothing
+ * but the data reloc inode references until relocation moves the pointers
+ * after writeback. Any other data RMW in a covered group is unexplained.
+ */
+static void rbio_count_rmw_reason(struct btrfs_raid_bio *rbio)
+{
+ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+ u8 idx = rbio->pad_reason;
+ bool reloc_writer = false;
+ struct bio *bio;
+
+ if (idx >= BTRFS_STRIPE_PAD_NR_REASONS)
+ idx = BTRFS_STRIPE_PAD_NR_REASONS; /* not tried */
+ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_by[idx]);
+ if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA))
+ return;
+
+ if (idx == BTRFS_STRIPE_PAD_NO_RUN) {
+ struct btrfs_block_group *bg;
+
+ if (rbio->pad_norun < BTRFS_STRIPE_NORUN_NR)
+ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_norun[rbio->pad_norun]);
+ bg = btrfs_lookup_block_group(fs_info,
+ rbio->bioc->full_stripe_logical);
+ if (bg) {
+ if (data_race(bg->ro))
+ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_norun_ro);
+ btrfs_put_block_group(bg);
+ }
+ }
+
+ /* The writer: the first bio's inode. */
+ spin_lock(&rbio->bio_list_lock);
+ bio = bio_list_peek(&rbio->bio_list);
+ if (bio && btrfs_bio(bio)->inode)
+ reloc_writer = btrfs_is_data_reloc_root(btrfs_bio(bio)->inode->root);
+ spin_unlock(&rbio->bio_list_lock);
+ /* Disjoint from the frontier count, so the two can be summed. */
+ if (reloc_writer && idx != BTRFS_STRIPE_PAD_UNCOVERED)
+ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_reloc_writer);
+}
+
/*
* Try to avoid the RMW read phase of a sub-stripe write by zero-filling the
* uncovered data sectors. Legal only where the stripe's run says a sector
bool ok = false;
int i;
+ rbio->pad_reason = RBIO_PAD_NOT_TRIED;
if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
return false;
}
if (!btrfs_stripe_run_pad_mask(fs_info, stripe_start, stripe_len, dead,
unwritten, &info)) {
+ rbio_record_pad(rbio, &info);
trace_btrfs_stripe_pad_decline(fs_info, stripe_start,
stripe_len, 0, 0, nsectors,
info.run_start, info.run_end,
if (!test_bit(i, dead)) {
atomic64_inc(&fs_info->stripe_park_stats.pad_decline_uncovered);
info.reason = BTRFS_STRIPE_PAD_UNCOVERED;
+ rbio_record_pad(rbio, &info);
trace_btrfs_stripe_pad_decline(fs_info, stripe_start,
stripe_len, 0, covered,
nsectors,
if (!ok)
return false;
+ /* The oracle agreed: only a page allocation failure refuses now. */
+ rbio->pad_reason = BTRFS_STRIPE_PAD_OK;
if (alloc_rbio_data_pages(rbio) < 0)
return false;
atomic64_inc(&fs_info->stripe_park_stats.data_rmw);
if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA))
atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_reads);
+ rbio_count_rmw_reason(rbio);
/* A parked-then-flushed write that still had to read. */
if (test_bit(RBIO_WAS_PARKED_BIT, &rbio->flags))
atomic64_inc(&fs_info->stripe_park_stats.rmw_reads);