#include "btrfs_inode.h"
#include "block-group.h"
#include "bio.h"
+#include "ordered-data.h"
/* set when additional merges to this rbio are not allowed */
#define RBIO_RMW_LOCKED_BIT 1
{
rbio->pad_reason = info->reason;
rbio->pad_norun = info->norun;
+#ifdef CONFIG_BTRFS_DEBUG
+ rbio->pad_run_open = info->run_open;
+ rbio->pad_run_class = info->run_class;
+ rbio->pad_run_start = info->run_start;
+ rbio->pad_run_end = info->run_end;
+ rbio->pad_run_offset = info->run_offset;
+#endif
}
static_assert(BTRFS_STRIPE_PAD_NR_REASONS + 1 ==
static_assert(BTRFS_STRIPE_NORUN_NR ==
ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_park_stats.data_rmw_norun));
+#ifdef CONFIG_BTRFS_DEBUG
+static_assert(BTRFS_STRIPE_PAD_NR_REASONS + 1 ==
+ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_rmw_audit_stats.hole_by));
+
+/* What the audit found under one RMW's uncovered sectors. */
+struct rmw_audit {
+ int foreign; /* sectors under extents another root owns */
+ int reloc; /* sectors under extents only relocation owns */
+ u64 min_gen;
+ u64 max_gen;
+ u64 first; /* first foreign extent */
+ u64 first_root; /* one other root referencing it (0: shared ref) */
+};
+
+/* One extent being classified: its uncovered sectors and its owners. */
+struct rmw_audit_ext {
+ u64 bytenr;
+ u64 gen;
+ int sectors;
+ int reloc_refs;
+ int other_refs; /* includes shared refs: owner unknown, so foreign */
+ u64 other_root;
+};
+
+static void rmw_audit_ext_ref(struct rmw_audit_ext *e, u64 root)
+{
+ if (root == BTRFS_DATA_RELOC_TREE_OBJECTID) {
+ e->reloc_refs++;
+ } else {
+ e->other_refs++;
+ if (!e->other_root)
+ e->other_root = root;
+ }
+}
+
+static void rmw_audit_ext_done(struct rmw_audit *a, struct rmw_audit_ext *e)
+{
+ if (!e->sectors)
+ goto out;
+ /* No ref seen at all: be conservative and call it foreign. */
+ if (e->other_refs || !e->reloc_refs) {
+ if (!a->foreign) {
+ a->first = e->bytenr;
+ a->first_root = e->other_root;
+ }
+ a->foreign += e->sectors;
+ a->min_gen = min(a->min_gen, e->gen);
+ a->max_gen = max(a->max_gen, e->gen);
+ } else {
+ a->reloc += e->sectors;
+ }
+out:
+ memset(e, 0, sizeof(*e));
+}
+
+/*
+ * Classify the data sectors of this rbio's stripe that the write does NOT
+ * cover by the COMMITTED extent under them and that extent's owners. Those
+ * are the sectors a torn RMW can destroy after a crash. An extent owned only
+ * by the data reloc tree is relocation's own orphaned copy, discarded by
+ * orphan cleanup if a crash comes before its pointers move; any other owner
+ * is a real write hole. Commit root, no locking, as in
+ * rmw_block_referenced(). Returns -1 when the lookup cannot answer.
+ *
+ * The commit root is switched shortly before the super lands, so an extent
+ * of the transaction committing right now can count early: this errs towards
+ * reporting a hole, never towards hiding one.
+ */
+static int rmw_committed_uncovered(struct btrfs_raid_bio *rbio,
+ struct rmw_audit *a)
+{
+ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+ const u64 stripe_start = rbio->bioc->full_stripe_logical;
+ const u64 stripe_end = stripe_start +
+ (u64)rbio->nr_data * rbio->bioc->stripe_len;
+ const u32 bits = fs_info->sectorsize_bits;
+ struct btrfs_root *extent_root = btrfs_extent_root(fs_info, stripe_start);
+ struct rmw_audit_ext e = { 0 };
+ struct btrfs_path *path;
+ struct btrfs_key key;
+ int ret;
+
+ memset(a, 0, sizeof(*a));
+ a->min_gen = U64_MAX;
+ /*
+ * The caller holds the full stripe lock, so nothing here may wait for
+ * something that can wait for this stripe: a tree block read that
+ * needs a raid56 rebuild of it (mixed block groups keep tree blocks in
+ * it, see fill_data_csums(), so they are not audited), or
+ * commit_root_sem, whose holder may be reading such a block. Read
+ * cached blocks only, and take the semaphore only if it is free. Each
+ * miss counts as an audit error.
+ */
+ if (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_METADATA)
+ return -1;
+ path = btrfs_alloc_path();
+ if (!path)
+ return -1;
+ path->search_commit_root = 1;
+ path->skip_locking = 1;
+ path->nowait = 1;
+
+ if (!down_read_trylock(&fs_info->commit_root_sem)) {
+ btrfs_free_path(path);
+ return -1;
+ }
+ key.objectid = stripe_start;
+ key.type = BTRFS_EXTENT_ITEM_KEY;
+ key.offset = 0;
+ ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+ if (ret < 0)
+ goto fail;
+ /* An extent starting below the stripe may reach into it. */
+ ret = btrfs_previous_extent_item(extent_root, path,
+ stripe_start > SZ_1G ? stripe_start - SZ_1G : 0);
+ if (ret < 0)
+ goto fail;
+ if (ret > 0) {
+ btrfs_release_path(path);
+ ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+ if (ret < 0)
+ goto fail;
+ }
+ while (1) {
+ struct extent_buffer *leaf = path->nodes[0];
+ const int slot = path->slots[0];
+ struct btrfs_extent_item *ei;
+ unsigned long ptr, end;
+ u64 lo, hi;
+
+ if (slot >= btrfs_header_nritems(leaf)) {
+ ret = btrfs_next_leaf(extent_root, path);
+ if (ret < 0)
+ goto fail;
+ if (ret > 0)
+ break;
+ continue;
+ }
+ btrfs_item_key_to_cpu(leaf, &key, slot);
+
+ /* Keyed backrefs of the extent being classified. */
+ if (e.sectors && key.objectid == e.bytenr) {
+ if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
+ struct btrfs_extent_data_ref *dref;
+
+ dref = btrfs_item_ptr(leaf, slot,
+ struct btrfs_extent_data_ref);
+ rmw_audit_ext_ref(&e, btrfs_extent_data_ref_root(leaf, dref));
+ } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
+ e.other_refs++;
+ }
+ path->slots[0]++;
+ continue;
+ }
+ if (key.type != BTRFS_EXTENT_ITEM_KEY) {
+ if (key.objectid >= stripe_end)
+ break;
+ path->slots[0]++;
+ continue;
+ }
+ rmw_audit_ext_done(a, &e);
+ if (key.objectid >= stripe_end)
+ break;
+ if (key.objectid + key.offset <= stripe_start) {
+ path->slots[0]++;
+ continue;
+ }
+ ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item);
+ lo = max(key.objectid, stripe_start);
+ hi = min(key.objectid + key.offset, stripe_end);
+ for (u64 b = lo; b < hi; b += fs_info->sectorsize)
+ if (rbio->bio_paddrs[((b - stripe_start) >> bits) * rbio->sector_nsteps] == INVALID_PADDR)
+ e.sectors++;
+ if (!e.sectors) {
+ path->slots[0]++;
+ continue;
+ }
+ e.bytenr = key.objectid;
+ e.gen = btrfs_extent_generation(leaf, ei);
+ /* Inline backrefs: data extents carry no tree_block_info. */
+ ptr = (unsigned long)(ei + 1);
+ end = (unsigned long)ei + btrfs_item_size(leaf, slot);
+ while (ptr < end) {
+ struct btrfs_extent_inline_ref *iref = (void *)ptr;
+ const int type = btrfs_extent_inline_ref_type(leaf, iref);
+ const u32 size = btrfs_extent_inline_ref_size(type);
+
+ if (type == BTRFS_EXTENT_DATA_REF_KEY) {
+ struct btrfs_extent_data_ref *dref =
+ (void *)&iref->offset;
+
+ rmw_audit_ext_ref(&e, btrfs_extent_data_ref_root(leaf, dref));
+ } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
+ e.other_refs++;
+ } else if (type != BTRFS_EXTENT_OWNER_REF_KEY) {
+ /* Not a data ref we understand: foreign. */
+ e.other_refs++;
+ break;
+ }
+ if (!size)
+ break;
+ ptr += size;
+ }
+ path->slots[0]++;
+ }
+ rmw_audit_ext_done(a, &e);
+ up_read(&fs_info->commit_root_sem);
+ btrfs_free_path(path);
+ return a->foreign + a->reloc;
+fail:
+ up_read(&fs_info->commit_root_sem);
+ btrfs_free_path(path);
+ return -1;
+}
+
+/*
+ * With /sys/fs/btrfs/<uuid>/debug/stripe_rmw_audit set: prove from the
+ * committed extent tree, not from the allocator's design, that a data RMW put
+ * no other owner's committed data at risk, and log the writer of any that
+ * did (and of every NO_RUN RMW, the class the allocator cannot explain by
+ * construction). @idx is the RMW's data_rmw_by index.
+ */
+static void rbio_audit_rmw(struct btrfs_raid_bio *rbio, u8 idx)
+{
+ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+ const u64 stripe_start = rbio->bioc->full_stripe_logical;
+ struct btrfs_block_group *bg;
+ struct rmw_audit a;
+ u64 root_id = 0, ino = 0, foff = 0, oflags = 0, oseq = 0, odisk = 0;
+ u64 bg_flags = 0;
+ bool orun = false;
+ int nbios = 0;
+ int ro = -1;
+ int ret;
+ struct bio *bio;
+
+ if (!READ_ONCE(fs_info->stripe_rmw_audit))
+ return;
+ atomic64_inc(&fs_info->stripe_rmw_audit_stats.audited);
+
+ spin_lock(&rbio->bio_list_lock);
+ bio_list_for_each(bio, &rbio->bio_list) {
+ struct btrfs_bio *bbio = btrfs_bio(bio);
+
+ if (nbios++)
+ continue;
+ if (!bbio->inode ||
+ btrfs_ino(bbio->inode) == BTRFS_BTREE_INODE_OBJECTID)
+ continue;
+ root_id = btrfs_root_id(bbio->inode->root);
+ ino = btrfs_ino(bbio->inode);
+ foff = bbio->file_offset;
+ if (bbio->ordered) {
+ oflags = bbio->ordered->flags;
+ oseq = bbio->ordered->stripe_seq;
+ odisk = bbio->ordered->disk_bytenr;
+ orun = bbio->ordered->stripe_run != NULL;
+ }
+ }
+ spin_unlock(&rbio->bio_list_lock);
+
+ /* The audit needs bio sectors mapped; idempotent. */
+ index_rbio_pages(rbio);
+ ret = rmw_committed_uncovered(rbio, &a);
+ if (ret < 0) {
+ atomic64_inc(&fs_info->stripe_rmw_audit_stats.audit_err);
+ } else if (a.foreign) {
+ atomic64_inc(&fs_info->stripe_rmw_audit_stats.hole_by[idx]);
+ atomic64_add(a.foreign, &fs_info->stripe_rmw_audit_stats.hole_sectors);
+ if (root_id == BTRFS_DATA_RELOC_TREE_OBJECTID)
+ atomic64_inc(&fs_info->stripe_rmw_audit_stats.reloc_writer_foreign);
+ } else if (a.reloc) {
+ atomic64_inc(&fs_info->stripe_rmw_audit_stats.hole_reloc_only);
+ }
+ if (ret > 0)
+ atomic64_add(a.reloc, &fs_info->stripe_rmw_audit_stats.hole_reloc_sectors);
+
+ if (idx == BTRFS_STRIPE_PAD_NO_RUN) {
+ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+ if (bg) {
+ ro = data_race(bg->ro);
+ bg_flags = bg->flags;
+ btrfs_put_block_group(bg);
+ }
+ }
+
+ /*
+ * Separate call sites, so separate rate limits: a flood of NO_RUN
+ * lines cannot starve the hole lines.
+ */
+ if (ret > 0 && a.foreign)
+ btrfs_warn_rl(fs_info,
+"rmw audit: write hole: stripe %llu reason %u norun %u foreign %d reloc %d gen %llu..%llu first %llu first_root %lld fsgen %llu committed %llu writer root %lld ino %llu foff %llu oe_flags 0x%llx oe_disk %llu oe_seq %llu oe_run %d nbios %d",
+ stripe_start, idx, rbio->pad_norun, a.foreign,
+ a.reloc, a.min_gen, a.max_gen, a.first,
+ (s64)a.first_root, btrfs_get_fs_generation(fs_info),
+ btrfs_get_last_trans_committed(fs_info),
+ (s64)root_id, ino, foff, oflags, odisk, oseq, orun,
+ nbios);
+ else if (idx == BTRFS_STRIPE_PAD_NO_RUN)
+ btrfs_info_rl(fs_info,
+"rmw audit: no_run RMW stripe %llu norun %u bg_ro %d bg_flags 0x%llx run %llu..%llu off %llu open %u class %u reloc_sectors %d fsgen %llu committed %llu writer root %lld ino %llu foff %llu oe_flags 0x%llx oe_seq %llu oe_run %d nbios %d",
+ stripe_start, rbio->pad_norun, ro, bg_flags,
+ rbio->pad_run_start, rbio->pad_run_end,
+ rbio->pad_run_offset, rbio->pad_run_open,
+ rbio->pad_run_class, ret > 0 ? a.reloc : 0,
+ btrfs_get_fs_generation(fs_info),
+ btrfs_get_last_trans_committed(fs_info),
+ (s64)root_id, ino, foff, oflags, oseq, orun, nbios);
+}
+#else
+static void rbio_audit_rmw(struct btrfs_raid_bio *rbio, u8 idx) { }
+#endif
+
/*
* Attribute a read-modify-write to why padding declined for this rbio. With
* allow_rmw empty, a data RMW in a stripe_alloc group is expected in two
/* Disjoint from the frontier count, so the two can be summed. */
if (reloc_writer && idx != BTRFS_STRIPE_PAD_UNCOVERED)
atomic64_inc(&fs_info->stripe_park_stats.data_rmw_reloc_writer);
+
+ rbio_audit_rmw(rbio, idx);
}
/*
#ifdef CONFIG_BTRFS_DEBUG
+/*
+ * Audit every data read-modify-write against the committed extent tree
+ * (raid56.c): a tree search per RMW, so off by default.
+ */
+static ssize_t btrfs_stripe_rmw_audit_show(struct kobject *kobj,
+ struct kobj_attribute *a, char *buf)
+{
+ struct btrfs_fs_info *fs_info = to_fs_info(get_btrfs_kobj(kobj));
+
+ return sysfs_emit(buf, "%d\n", READ_ONCE(fs_info->stripe_rmw_audit));
+}
+
+static ssize_t btrfs_stripe_rmw_audit_store(struct kobject *kobj,
+ struct kobj_attribute *a,
+ const char *buf, size_t len)
+{
+ struct btrfs_fs_info *fs_info = to_fs_info(get_btrfs_kobj(kobj));
+ bool val;
+ int ret;
+
+ ret = kstrtobool(buf, &val);
+ if (ret)
+ return ret;
+ WRITE_ONCE(fs_info->stripe_rmw_audit, val);
+ return len;
+}
+BTRFS_ATTR_RW(debug_mount, stripe_rmw_audit, btrfs_stripe_rmw_audit_show,
+ btrfs_stripe_rmw_audit_store);
+
+static ssize_t btrfs_stripe_rmw_audit_stats_show(struct kobject *kobj,
+ struct kobj_attribute *a,
+ char *buf)
+{
+ struct btrfs_fs_info *fs_info = to_fs_info(get_btrfs_kobj(kobj));
+
+ return sysfs_emit(buf,
+ "data_rmw_audited %lld\n"
+ "data_rmw_hole_no_run %lld\n"
+ "data_rmw_hole_uncovered %lld\n"
+ "data_rmw_hole_other %lld\n"
+ "data_rmw_hole_sectors %lld\n"
+ "data_rmw_hole_reloc_only %lld\n"
+ "data_rmw_hole_reloc_sectors %lld\n"
+ "data_rmw_reloc_writer_foreign %lld\n"
+ "data_rmw_audit_err %lld\n",
+ atomic64_read(&fs_info->stripe_rmw_audit_stats.audited),
+ atomic64_read(&fs_info->stripe_rmw_audit_stats.hole_by[BTRFS_STRIPE_PAD_NO_RUN]),
+ atomic64_read(&fs_info->stripe_rmw_audit_stats.hole_by[BTRFS_STRIPE_PAD_UNCOVERED]),
+ atomic64_read(&fs_info->stripe_rmw_audit_stats.hole_by[BTRFS_STRIPE_PAD_OK]) +
+ atomic64_read(&fs_info->stripe_rmw_audit_stats.hole_by[BTRFS_STRIPE_PAD_NR_REASONS]),
+ atomic64_read(&fs_info->stripe_rmw_audit_stats.hole_sectors),
+ atomic64_read(&fs_info->stripe_rmw_audit_stats.hole_reloc_only),
+ atomic64_read(&fs_info->stripe_rmw_audit_stats.hole_reloc_sectors),
+ atomic64_read(&fs_info->stripe_rmw_audit_stats.reloc_writer_foreign),
+ atomic64_read(&fs_info->stripe_rmw_audit_stats.audit_err));
+}
+BTRFS_ATTR(debug_mount, stripe_rmw_audit_stats,
+ btrfs_stripe_rmw_audit_stats_show);
+
/*
* Per-filesystem runtime debugging exported via sysfs.
*
* Path: /sys/fs/btrfs/UUID/debug/
*/
static const struct attribute *btrfs_debug_mount_attrs[] = {
+ BTRFS_ATTR_PTR(debug_mount, stripe_rmw_audit),
+ BTRFS_ATTR_PTR(debug_mount, stripe_rmw_audit_stats),
NULL,
};