return true;
}
+/*
+ * For the raid56 pad-to-full optimization: a sub-stripe write into a
+ * stripe covered by a live stripe run may zero-fill (instead of read) any
+ * sector at or past the run's allocation frontier, because such sectors
+ * have never been written in the run's lifetime and, once the run closes,
+ * its stripe's tail cannot be reallocated while the stripe holds live
+ * data. Returns true and sets *pad_from to the frontier when the stripe
+ * at [stripe_start, stripe_start + stripe_len) has a paddable tail. The
+ * frontier can grow right after this returns; that is safe, because the
+ * later allocation's write serializes behind the caller's stripe lock
+ * and overwrites the zeros.
+ */
+bool btrfs_stripe_run_pad_start(struct btrfs_fs_info *fs_info,
+ u64 stripe_start, u64 stripe_len,
+ u64 *pad_from)
+{
+ struct btrfs_block_group *bg;
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ bool ret = false;
+
+ if (list_empty_careful(&fs_info->open_stripe_bgs))
+ return false;
+ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+ if (!bg)
+ return false;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (stripe_start < run->start || stripe_start >= run->end)
+ continue;
+ if (run->offset > stripe_start &&
+ run->offset < stripe_start + stripe_len) {
+ *pad_from = run->offset;
+ ret = true;
+ }
+ break;
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ btrfs_put_block_group(bg);
+ return ret;
+}
+
/* No open or draining run covers @bytenr any more. */
static bool stripe_log_range_settled(struct btrfs_block_group *bg, u64 bytenr)
{
*/
#define RBIO_PARKED_BIT 4
+/*
+ * Set when a partial write rbio's uncovered data sectors were zero-filled
+ * instead of read (they lie at or past the covering stripe run's
+ * allocation frontier and have never been written). The write path then
+ * writes them out with the stripe -- parity must match the disk, or
+ * scrub would flag the never-written sectors -- turning the partial
+ * write into a full-stripe write with no read phase.
+ */
+#define RBIO_PADDED_BIT 5
+
#define RBIO_CACHE_SIZE 1024
/*
continue;
if (stripe < rbio->nr_data) {
- paddrs = sector_paddrs_in_rbio(rbio, stripe, sectornr, 1);
+ /*
+ * A padded rbio writes its zero-filled stripe sectors
+ * too, so parity matches the disk (see RBIO_PADDED_BIT).
+ */
+ paddrs = sector_paddrs_in_rbio(rbio, stripe, sectornr,
+ !test_bit(RBIO_PADDED_BIT, &rbio->flags));
if (paddrs == NULL)
continue;
} else {
}
}
+/*
+ * Try to avoid the RMW read phase of a sub-stripe write by zero-filling
+ * the uncovered data sectors. Legal only when every uncovered sector
+ * lies at or past its stripe run's allocation frontier -- never written
+ * in the run's lifetime, and unreachable by other allocations while this
+ * stripe holds live data (see btrfs_stripe_run_pad_start()). A frontier
+ * that grows under us is safe: the newer allocation's write serializes
+ * behind this rbio's stripe lock and lands over the zeros.
+ */
+static bool rmw_try_pad_full(struct btrfs_raid_bio *rbio)
+{
+ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+ const u64 stripe_start = rbio->bioc->full_stripe_logical;
+ const u32 sectorsize = fs_info->sectorsize;
+ const u64 stripe_len = (u64)rbio->nr_data * BTRFS_STRIPE_LEN;
+ u64 pad_from;
+ int i;
+
+ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+ return false;
+ if (!btrfs_stripe_run_pad_start(fs_info, stripe_start, stripe_len,
+ &pad_from))
+ return false;
+
+ /* Populate bio_paddrs so covered sectors are visible below. */
+ index_rbio_pages(rbio);
+
+ /* Every data sector this rbio does not cover must be paddable. */
+ for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) {
+ const int stripe_nr = i / rbio->stripe_nsectors;
+ const int sectornr = i % rbio->stripe_nsectors;
+ u64 logical = stripe_start + stripe_nr * BTRFS_STRIPE_LEN +
+ (u64)sectornr * sectorsize;
+
+ if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
+ continue;
+ if (logical < pad_from)
+ return false;
+ }
+
+ if (alloc_rbio_data_pages(rbio) < 0)
+ return false;
+
+ for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) {
+ const int stripe_nr = i / rbio->stripe_nsectors;
+ const int sectornr = i % rbio->stripe_nsectors;
+ int step;
+
+ if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
+ continue;
+ for (step = 0; step < rbio->sector_nsteps; step++) {
+ phys_addr_t paddr = rbio_stripe_paddr(rbio, stripe_nr,
+ sectornr, step);
+ void *kaddr = kmap_local_paddr(paddr);
+
+ memset(kaddr, 0, min_t(u32, sectorsize, PAGE_SIZE));
+ kunmap_local(kaddr);
+ }
+ set_bit(i, rbio->stripe_uptodate_bitmap);
+ }
+
+ set_bit(RBIO_PADDED_BIT, &rbio->flags);
+ /* The whole stripe goes out: every vertical position is written. */
+ bitmap_set(&rbio->dbitmap, 0, rbio->stripe_nsectors);
+ return true;
+}
+
/*
* To determine if we need to read any sector from the disk.
* Should only be utilized in RMW path, to skip cached rbio.
/*
* Either full stripe write, or we have every data sector already
* cached, can go to write path immediately.
+ *
+ * Padding must win over the cache: a stolen cached rbio can carry
+ * all-uptodate pages from the stripe's PREVIOUS life -- before its
+ * extents were freed and the stripe was re-claimed -- and letting
+ * those pages skip the pad path bakes the cache's memory of dead
+ * data into fresh parity while only the bio sectors reach disk.
+ * If any on-disk column then diverges from that memory (device
+ * corruption, or simply free-space columns nobody rewrote), the
+ * stripe is latently unreconstructible. A paddable rbio instead
+ * zero-fills and WRITES every uncovered column, making the stripe
+ * self-consistent on disk regardless of what the cache remembered;
+ * the claimed-stripe invariant (a stripe write depends on nothing
+ * on disk) requires this ordering.
*/
- if (!rbio_is_full(rbio) && need_read_stripe_sectors(rbio)) {
+ if (!rbio_is_full(rbio) && !rmw_try_pad_full(rbio) &&
+ need_read_stripe_sectors(rbio)) {
/*
* Now we're doing sub-stripe write, also need all data stripes
* to do the full RMW.
projects / linux / commitdiff