wait_var_event(&bg->reservations, !atomic_read(&bg->reservations));
}
- - if (bg->start == READ_ONCE(fs_info->data_reloc_bg))
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Open stripe runs
++++++++++++ ++ ++ ++++++++ * ================
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * The raid56 stripe allocation policy only allocates from "open stripe
++++++++++++ ++ ++ ++++++++ * runs": contiguous, stripe-aligned regions claimed whole from the free
++++++++++++ ++ ++ ++++++++ * space cache (btrfs_claim_free_stripe_run()) and filled strictly
++++++++++++ ++ ++ ++++++++ * sequentially. A run is closed when it is exhausted, when it loses a
++++++++++++ ++ ++ ++++++++ * band collision (stripe_run_place()), when a private run is too small and
++++++++++++ ++ ++ ++++++++ * is replaced, when a metadata run belongs to an earlier transaction, or
++++++++++++ ++ ++ ++++++++ * when it is retired at transaction commit. A closed run is never
++++++++++++ ++ ++ ++++++++ * reopened, and its unallocated tail returns to the free space cache
++++++++++++ ++ ++ ++++++++ * where, being part of a partially filled stripe, it cannot be claimed
++++++++++++ ++ ++ ++++++++ * until the whole stripe frees. So every stripe is written within one
++++++++++++ ++ ++ ++++++++ * run's lifetime, which closes the raid56 write hole for these block
++++++++++++ ++ ++ ++++++++ * groups.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Each run counts the bytes reserved from it whose IO has not completed
++++++++++++ ++ ++ ++++++++ * (inflight_bytes). At commit, while the transaction still accepts joins
++++++++++++ ++ ++ ++++++++ * (before TRANS_STATE_COMMIT_DOING), btrfs_retire_open_stripes() closes
++++++++++++ ++ ++ ++++++++ * the data runs opened before the commit and waits for them to drain. The
++++++++++++ ++ ++ ++++++++ * wait must be a pure data-IO wait, never an ordered extent completion
++++++++++++ ++ ++ ++++++++ * wait, and must happen where joins still succeed: a task holding
++++++++++++ ++ ++ ++++++++ * unreported run bytes may block on a join (e.g. a chunk allocation during
++++++++++++ ++ ++ ++++++++ * delalloc). Metadata runs are retired after tree writeback
++++++++++++ ++ ++ ++++++++ * (btrfs_retire_meta_stripes()); persistent NOCOW runs survive commits.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Runs opened while the commit is in progress carry a newer open_seq and
++++++++++++ ++ ++ ++++++++ * belong to the next window. An ordered extent from such a run that
++++++++++++ ++ ++ ++++++++ * completes before the commit ends sees that its run's open_seq is not
++++++++++++ ++ ++ ++++++++ * retired by this transaction and defers its file extent to the next one.
++++++++++++ ++ ++ ++++++++ * The membership add, the sequence read and the run installation happen
++++++++++++ ++ ++ ++++++++ * atomically under fs_info->open_stripe_lock, so every run is either seen
++++++++++++ ++ ++ ++++++++ * by the walk that retires its window or stamped with a sequence that
++++++++++++ ++ ++ ++++++++ * forces the deferral.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Lock order: fs_info->open_stripe_lock outside bg->stripe_run_lock. The
++++++++++++ ++ ++ ++++++++ * stripe_run_lock is taken with interrupts disabled, which no current
++++++++++++ ++ ++ ++++++++ * caller needs: completed IO is reported from bio completion work, which
++++++++++++ ++ ++ ++++++++ * runs in task context. Membership in fs_info->open_stripe_bgs holds a
++++++++++++ ++ ++ ++++++++ * block group reference, dropped by the retire walk once the group has no
++++++++++++ ++ ++ ++++++++ * runs left, or by btrfs_free_open_stripe_runs() at teardown.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++static bool stripe_extents_owned_by(struct btrfs_fs_info *fs_info,
++++++++++++ ++ ++ ++++++++ u64 root_id, u64 ino, u64 start, u64 len,
++++++++++++ ++ ++ ++++++++ u64 *frontier, bool nowait);
++++++++++++ ++ ++ ++++++++static bool stripe_range_run_free(struct btrfs_block_group *bg, u64 start,
++++++++++++ ++ ++ ++++++++ u64 len);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++struct btrfs_open_stripe_run {
++++++++++++ ++ ++ ++++++++ struct list_head list; /* bg->open_stripe_runs */
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ u64 start;
++++++++++++ ++ ++ ++++++++ u64 end; /* exclusive; shrunk to offset on close */
++++++++++++ ++ ++ ++++++++ u64 offset; /* next unallocated byte */
++++++++++++ ++ ++ ++++++++ u64 inflight_bytes; /* reserved bytes with data IO pending */
++++++++++++ ++ ++ ++++++++ u64 open_seq; /* fs_info->stripe_retire_seq at open */
++++++++++++ ++ ++ ++++++++ u64 transid; /* fs_info->generation at open */
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Private runs are found by (root, owner): root is the subvolume's
++++++++++++ ++ ++ ++++++++ * root id, owner the inode number (LOG, NOCOW), a shared LOG key
++++++++++++ ++ ++ ++++++++ * (bit 63 set, see stripe_run_owner()), or for META_LOG the root id
++++++++++++ ++ ++ ++++++++ * again. Inode numbers are only unique within a subvolume.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ u64 root;
++++++++++++ ++ ++ ++++++++ u64 owner;
++++++++++++ ++ ++ ++++++++ enum btrfs_stripe_run_class class;
++++++++++++ ++ ++ ++++++++ bool open; /* accepting allocations */
++++++++++++ ++ ++ ++++++++ bool persistent; /* survives commits; see close_block_group_stripe_runs() */
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Relocation's run: kept after it closes and drains, until relocation
++++++++++++ ++ ++ ++++++++ * releases it (btrfs_release_stripe_reloc_runs()). See
++++++++++++ ++ ++ ++++++++ * run_freed_on_drain().
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ bool retained;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * One bit per sector of the run, set while the sector holds an
++++++++++++ ++ ++ ++++++++ * allocation someone still intends to write. Padding uses it to find
++++++++++++ ++ ++ ++++++++ * dead sectors below the frontier (allocations abandoned before they
++++++++++++ ++ ++ ++++++++ * were written), which are as safe to zero-fill as space above it.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ unsigned long *live;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Sectors of @live whose data has been written (raid56 write
++++++++++++ ++ ++ ++++++++ * completions, btrfs_stripe_run_mark_written()); same size as @live.
++++++++++++ ++ ++ ++++++++ * A stripe with live sectors not yet written is still waiting for
++++++++++++ ++ ++ ++++++++ * writes, and its cached rbio stays pinned (raid56.c, cache_rbio()).
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ unsigned long *written;
++++++++++++ ++ ++ ++++++++ u32 live_nbits; /* sectors @live covers; caps growth */
++++++++++++ ++ ++ ++++++++ /* META_LOG: the owning subvolume's log_transid this run belongs to. */
++++++++++++ ++ ++ ++++++++ u64 log_transid;
++++++++++++ ++ ++ ++++++++};
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++static void btrfs_dump_open_stripe_runs(struct btrfs_block_group *bg)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list)
++++++++++++ ++ ++ ++++++++ btrfs_warn(bg->fs_info,
++++++++++++ ++ ++ ++++++++" run %llu..%llu class %d frontier %llu open %d inflight %llu transid %llu",
++++++++++++ ++ ++ ++++++++ run->start, run->end, run->class, run->offset,
++++++++++++ ++ ++ ++++++++ run->open, run->inflight_bytes, run->transid);
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++static bool bg_has_undrained_meta_runs(struct btrfs_block_group *bg,
++++++++++++ ++ ++ ++++++++ u64 transid)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ bool ret = false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (!btrfs_stripe_run_class_is_meta(run->class))
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * An open run here belongs to the running transaction: the
++++++++++++ ++ ++ ++++++++ * retirer closed every run it is responsible for before its
++++++++++++ ++ ++ ++++++++ * wait, so this one was opened while it waited, and drains
++++++++++++ ++ ++ ++++++++ * when that transaction's own commit writes its tree blocks.
++++++++++++ ++ ++ ++++++++ * Waiting on it would be waiting for writes this commit never
++++++++++++ ++ ++ ++++++++ * issued.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (run->open)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Likewise a run the running transaction opened and a band
++++++++++++ ++ ++ ++++++++ * collision or eviction closed early: its blocks are written,
++++++++++++ ++ ++ ++++++++ * and its stripes settled, by that transaction's own commit.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (run->transid > transid)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (run->inflight_bytes) {
++++++++++++ ++ ++ ++++++++ ret = true;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++static bool open_stripe_run_drained(const struct btrfs_open_stripe_run *run)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ return !run->open && run->inflight_bytes == 0;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Round to a full stripe.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * full_stripe_len is nr_data_stripes * stripe_len, which is not a power of
++++++++++++ ++ ++ ++++++++ * two for most raid56 geometries (3 data stripes: 192K), so the mask-based
++++++++++++ ++ ++ ++++++++ * round_down(), round_up() and IS_ALIGNED() must not be used. Divide
++++++++++++ ++ ++ ++++++++ * instead.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static u64 stripe_round_down(u64 val, u64 fsl)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ return div64_u64(val, fsl) * fsl;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++static u64 stripe_round_up(u64 val, u64 fsl)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ return stripe_round_down(val + fsl - 1, fsl);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++static bool stripe_is_aligned(u64 val, u64 fsl)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ return stripe_round_down(val, fsl) == val;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Band for a run with @remaining bytes of unallocated space: the power-of-two
++++++++++++ ++ ++ ++++++++ * bucket of its remaining sectors (band k == remaining in [2^k, 2^(k+1))
++++++++++++ ++ ++ ++++++++ * sectors). Runs are kept at most one per band so an allocation can take the
++++++++++++ ++ ++ ++++++++ * smallest band that fits.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static int stripe_run_band(const struct btrfs_block_group *bg, u64 remaining)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ u32 sectors = remaining >> bg->fs_info->sectorsize_bits;
++++++++++++ ++ ++ ++++++++ int band;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!sectors)
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++ band = fls(sectors) - 1;
++++++++++++ ++ ++ ++++++++ if (band >= BTRFS_STRIPE_RUN_NR_BANDS)
++++++++++++ ++ ++ ++++++++ band = BTRFS_STRIPE_RUN_NR_BANDS - 1;
++++++++++++ ++ ++ ++++++++ return band;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Growth headroom for a run's liveness map, in full stripes.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * The map is sized once and never reallocated, because the only place a run
++++++++++++ ++ ++ ++++++++ * grows -- the growable frontier in btrfs_alloc_from_open_stripe() -- holds
++++++++++++ ++ ++ ++++++++ * bg->stripe_run_lock with interrupts disabled, where nothing may sleep.
++++++++++++ ++ ++ ++++++++ * Sizing it beyond the run's initial span is what lets that path work at
++++++++++++ ++ ++ ++++++++ * all; a run that would outgrow its map stops growing instead, and the
++++++++++++ ++ ++ ++++++++ * allocation opens a fresh run.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++#define BTRFS_STRIPE_RUN_LIVE_STRIPES 32
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Allocate a run's liveness map. Must be called before taking
++++++++++++ ++ ++ ++++++++ * bg->stripe_run_lock: this sleeps. Sets *nbits to what the map covers.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static unsigned long *alloc_run_live_map(struct btrfs_block_group *bg,
++++++++++++ ++ ++ ++++++++ u64 run_len, u32 *nbits)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ const u64 span = max_t(u64, run_len,
++++++++++++ ++ ++ ++++++++ (u64)BTRFS_STRIPE_RUN_LIVE_STRIPES *
++++++++++++ ++ ++ ++++++++ bg->full_stripe_len);
++++++++++++ ++ ++ ++++++++ unsigned long *map;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ *nbits = span >> bg->fs_info->sectorsize_bits;
++++++++++++ ++ ++ ++++++++ map = bitmap_zalloc(*nbits, GFP_NOFS);
++++++++++++ ++ ++ ++++++++ if (!map)
++++++++++++ ++ ++ ++++++++ *nbits = 0;
++++++++++++ ++ ++ ++++++++ return map;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/* Mark [start, start+len) live (allocated) or dead (abandoned). */
++++++++++++ ++ ++ ++++++++static void run_live_set(struct btrfs_open_stripe_run *run, u64 start, u64 len,
++++++++++++ ++ ++ ++++++++ bool live)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
++++++++++++ ++ ++ ++++++++ u32 first, nr;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!run->live || !run->live_nbits || start < run->start)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ first = (start - run->start) >> bits;
++++++++++++ ++ ++ ++++++++ nr = len >> bits;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Growth is capped at the map size, so this should be unreachable. If
++++++++++++ ++ ++ ++++++++ * it happens, retire the map: btrfs_stripe_run_pad_mask() then declines
++++++++++++ ++ ++ ++++++++ * the run, and the write falls back to read-modify-write instead of
++++++++++++ ++ ++ ++++++++ * zero-filling a sector that may be live.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (unlikely(first + nr > run->live_nbits)) {
++++++++++++ ++ ++ ++++++++ WARN_ON_ONCE(1);
++++++++++++ ++ ++ ++++++++ run->live_nbits = 0;
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (live)
++++++++++++ ++ ++ ++++++++ bitmap_set(run->live, first, nr);
++++++++++++ ++ ++ ++++++++ else
++++++++++++ ++ ++ ++++++++ bitmap_clear(run->live, first, nr);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * A run is dead when every byte it handed out has been freed again
++++++++++++ ++ ++ ++++++++ * (btrfs_stripe_run_note_free() clears the live bits). It then protects no
++++++++++++ ++ ++ ++++++++ * data, and the whole free stripes inside its range stay unclaimable
++++++++++++ ++ ++ ++++++++ * (btrfs_stripe_run_range_usable()) until it is closed.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Requires inflight_bytes == 0, since an in-flight write must keep its
++++++++++++ ++ ++ ++++++++ * stripes, and a valid live map (live_nbits == 0 means liveness is
++++++++++++ ++ ++ ++++++++ * unknown). Caller holds bg->stripe_run_lock.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static bool open_stripe_run_dead(const struct btrfs_open_stripe_run *run)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ if (run->inflight_bytes)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ if (run->offset <= run->start)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ if (!run->live || !run->live_nbits)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ return bitmap_empty(run->live, run->live_nbits);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Whether a run is freed as soon as it drains. Two kinds must outlive the
++++++++++++ ++ ++ ++++++++ * drain, because later writes to their stripes still need a covering run to
++++++++++++ ++ ++ ++++++++ * park against or pad from:
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * - Metadata runs: writeback may submit a stripe's tree blocks as separate
++++++++++++ ++ ++ ++++++++ * rbios, so the run drains after the first one. They are freed by
++++++++++++ ++ ++ ++++++++ * btrfs_retire_meta_stripes() once every tree block is on disk.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * - Relocation runs: relocation preallocates its destination extents (no
++++++++++++ ++ ++ ++++++++ * IO, so the run drains at once) and writes them only after the commit
++++++++++++ ++ ++ ++++++++ * that ends relocate_block_group(). They are retained until relocation
++++++++++++ ++ ++ ++++++++ * releases them (btrfs_release_stripe_reloc_runs()). Meanwhile their
++++++++++++ ++ ++ ++++++++ * stripes hold only relocation's own copies, and the closed run keeps its
++++++++++++ ++ ++ ++++++++ * range unclaimable.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static bool run_freed_on_drain(const struct btrfs_open_stripe_run *run)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ return !btrfs_stripe_run_class_is_meta(run->class) && !run->retained;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++static void free_open_stripe_run(struct btrfs_block_group *bg,
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ ASSERT(open_stripe_run_drained(run));
++++++++++++ ++ ++ ++++++++ list_del(&run->list);
++++++++++++ ++ ++ ++++++++ bitmap_free(run->live);
++++++++++++ ++ ++ ++++++++ bitmap_free(run->written);
++++++++++++ ++ ++ ++++++++ kfree(run);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Wholly free stripes inside this run's range were counted as trapped
++++++++++++ ++ ++ ++++++++ * by the last stripe_unusable scan (btrfs_stripe_run_stranded_stripes());
++++++++++++ ++ ++ ++++++++ * with the run gone they are claimable again, so have the next commit
++++++++++++ ++ ++ ++++++++ * rescan the group and credit them back.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (READ_ONCE(bg->stripe_unusable_ready)) {
++++++++++++ ++ ++ ++++++++ set_bit(BLOCK_GROUP_FLAG_STRIPE_UNUSABLE_DIRTY, &bg->runtime_flags);
++++++++++++ ++ ++ ++++++++ atomic64_inc(&bg->fs_info->stripe_run_stats.strand_rescan_marks);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * A run has drained. Free it if nothing needs it any more; otherwise
++++++++++++ ++ ++ ++++++++ * (metadata or retained relocation runs) wake the waiters directly, since a
++++++++++++ ++ ++ ++++++++ * report can arrive after btrfs_retire_meta_stripes() started waiting, e.g.
++++++++++++ ++ ++ ++++++++ * when an unwritten tree block is freed.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static void open_stripe_run_drain_reported(struct btrfs_block_group *bg,
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ if (!open_stripe_run_drained(run))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ if (run_freed_on_drain(run))
++++++++++++ ++ ++ ++++++++ free_open_stripe_run(bg, run);
++++++++++++ ++ ++ ++++++++ else
++++++++++++ ++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Close a run: no further allocation may join it, and its unallocated tail
++++++++++++ ++ ++ ++++++++ * must be returned to the free space cache by the caller (after dropping the
++++++++++++ ++ ++ ++++++++ * lock). The caller has already removed @run from bg->open_stripe[][].
++++++++++++ ++ ++ ++++++++ * Caller holds bg->stripe_run_lock. Returns the tail length, sets *tail_start.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static u64 close_open_stripe_run(struct btrfs_block_group *bg,
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run,
++++++++++++ ++ ++ ++++++++ u64 *tail_start)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ u64 tail_len;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ ASSERT(run->open);
++++++++++++ ++ ++ ++++++++ run->open = false;
++++++++++++ ++ ++ ++++++++ *tail_start = run->offset;
++++++++++++ ++ ++ ++++++++ tail_len = run->end - run->offset;
++++++++++++ ++ ++ ++++++++ run->end = run->offset;
++++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= tail_len;
++++++++++++ ++ ++ ++++++++ if (open_stripe_run_drained(run) && !run->retained)
++++++++++++ ++ ++ ++++++++ free_open_stripe_run(bg, run);
++++++++++++ ++ ++ ++++++++ return tail_len;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Fold this block group's open-run remainder delta into the space_info
++++++++++++ ++ ++ ++++++++ * counter that reservations see. Bytes inside an open run are invisible
++++++++++++ ++ ++ ++++++++ * to the free space cache (the claim removed them) and will either be
++++++++++++ ++ ++ ++++++++ * allocated or become trapped tails when the run closes, so the data
++++++++++++ ++ ++ ++++++++ * reservation path must not admit writes against them: a reservation
++++++++++++ ++ ++ ++++++++ * that cannot allocate at writeback time discards the pages silently.
++++++++++++ ++ ++ ++++++++ * Called after every remainder mutation. Compute and apply are one
++++++++++++ ++ ++ ++++++++ * critical section (sinfo->lock nests inside stripe_run_lock): applied
++++++++++++ ++ ++ ++++++++ * out of order, a negative delta could overtake the positive it was
++++++++++++ ++ ++ ++++++++ * computed against and transiently underflow the space_info sum.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static void stripe_open_remainder_sync(struct btrfs_block_group *bg)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_space_info *sinfo = bg->space_info;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ s64 delta;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ delta = (s64)(bg->stripe_open_remainder - bg->stripe_open_reported);
++++++++++++ ++ ++ ++++++++ bg->stripe_open_reported = bg->stripe_open_remainder;
++++++++++++ ++ ++ ++++++++ if (delta && sinfo) {
++++++++++++ ++ ++ ++++++++ spin_lock(&sinfo->lock);
++++++++++++ ++ ++ ++++++++ if (delta < 0 && sinfo->bytes_stripe_open < (u64)-delta) {
++++++++++++ ++ ++ ++++++++ WARN_ON_ONCE(1);
++++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_open = 0;
++++++++++++ ++ ++ ++++++++ } else {
++++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_open += delta;
++++++++++++ ++ ++ ++++++++ }
+++++++++++++++ +++++++++++ /*
+++++++++++++++ +++++++++++ * A claimed run's remainder left the free space cache, but
+++++++++++++++ +++++++++++ * reloc_reserve_free() still counts it: move it out of a held
+++++++++++++++ +++++++++++ * group's share with it, or bytes_stripe_open and
+++++++++++++++ +++++++++++ * bytes_reloc_reserve both count it until the next commit's
+++++++++++++++ +++++++++++ * resync. Every writer of the share holds sinfo->lock.
+++++++++++++++ +++++++++++ */
+++++++++++++++ +++++++++++ btrfs_reloc_reserve_adjust(bg, -delta);
++++++++++++ ++ ++ ++++++++ spin_unlock(&sinfo->lock);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Install @run (not currently in any band slot) into its remaining-space band.
++++++++++++ ++ ++ ++++++++ * If a run already occupies that band, keep whichever has more remaining -- it
++++++++++++ ++ ++ ++++++++ * can serve larger future allocations, and closing the other traps less -- and
++++++++++++ ++ ++ ++++++++ * close the loser, returning its tail via *tail_start. Caller holds
++++++++++++ ++ ++ ++++++++ * bg->stripe_run_lock. Returns the closed tail length, or 0 if no collision.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static u64 stripe_run_place(struct btrfs_block_group *bg,
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run, u64 *tail_start)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ int class = run->class;
++++++++++++ ++ ++ ++++++++ int band = stripe_run_band(bg, run->end - run->offset);
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *occ = bg->open_stripe[class][band];
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Private runs (LOG, NOCOW, META_LOG) never occupy a band slot. */
++++++++++++ ++ ++ ++++++++ ASSERT(class != BTRFS_STRIPE_RUN_LOG);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (occ) {
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *keep, *shut;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (run->end - run->offset >= occ->end - occ->offset) {
++++++++++++ ++ ++ ++++++++ keep = run;
++++++++++++ ++ ++ ++++++++ shut = occ;
++++++++++++ ++ ++ ++++++++ } else {
++++++++++++ ++ ++ ++++++++ keep = occ;
++++++++++++ ++ ++ ++++++++ shut = run;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = keep;
++++++++++++ ++ ++ ++++++++ return close_open_stripe_run(bg, shut, tail_start);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = run;
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Allocate num_bytes from the block group's shared open runs of @class,
++++++++++++ ++ ++ ++++++++ * best fit across the remaining-space bands, claiming a new run via
++++++++++++ ++ ++ ++++++++ * btrfs_claim_free_stripe_run() when none fits. If the freshly claimed
++++++++++++ ++ ++ ++++++++ * stripes directly follow an open run of the same class, that run grows to
++++++++++++ ++ ++ ++++++++ * absorb them instead (see the comment at the claim site). The allocation
++++++++++++ ++ ++ ++++++++ * counts toward the run's inflight bytes until its IO is reported done or
++++++++++++ ++ ++ ++++++++ * abandoned.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in
++++++++++++ ++ ++ ++++++++ * this block group can satisfy the allocation, or -ENOMEM.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++int btrfs_alloc_from_open_stripe(struct btrfs_block_group *bg, u64 num_bytes,
++++++++++++ ++ ++ ++++++++ enum btrfs_stripe_run_class class,
++++++++++++ ++ ++ ++++++++ u64 *ret_offset, u64 *available)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
++++++++++++ ++ ++ ++++++++ const u64 transid = btrfs_get_fs_generation(fs_info);
++++++++++++ ++ ++ ++++++++ u64 evict_start[BTRFS_STRIPE_RUN_NR_BANDS];
++++++++++++ ++ ++ ++++++++ u64 evict_len[BTRFS_STRIPE_RUN_NR_BANDS];
++++++++++++ ++ ++ ++++++++ int nr_evict = 0;
++++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *new_run;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ u64 tail_start = 0;
++++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
++++++++++++ ++ ++ ++++++++ u64 start;
++++++++++++ ++ ++ ++++++++ u64 len;
++++++++++++ ++ ++ ++++++++ int band;
++++++++++++ ++ ++ ++++++++ int ret;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ ASSERT(num_bytes);
++++++++++++ ++ ++ ++++++++ *available = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Best fit: take the run in the smallest band that can hold this
++++++++++++ ++ ++ ++++++++ * allocation, and track the largest remaining across all bands as the
++++++++++++ ++ ++ ++++++++ * retry hint should nothing fit.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ run = NULL;
++++++++++++ ++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) {
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *r = bg->open_stripe[class][band];
++++++++++++ ++ ++ ++++++++ u64 rem;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!r)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Metadata runs are closed after the next transaction is
++++++++++++ ++ ++ ++++++++ * already running (btrfs_retire_meta_stripes()), so a slot can
++++++++++++ ++ ++ ++++++++ * still hold the previous transaction's run. Joining it would
++++++++++++ ++ ++ ++++++++ * put two transactions' tree blocks in the same stripes, and
++++++++++++ ++ ++ ++++++++ * the later one's writeback would read-modify-write parity
++++++++++++ ++ ++ ++++++++ * over the earlier one's committed blocks: the write hole.
++++++++++++ ++ ++ ++++++++ * Close it; its tail returns to the free space cache below.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_META && r->transid != transid) {
++++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
++++++++++++ ++ ++ ++++++++ evict_len[nr_evict] =
++++++++++++ ++ ++ ++++++++ close_open_stripe_run(bg, r,
++++++++++++ ++ ++ ++++++++ &evict_start[nr_evict]);
++++++++++++ ++ ++ ++++++++ if (evict_len[nr_evict])
++++++++++++ ++ ++ ++++++++ nr_evict++;
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ rem = r->end - r->offset;
++++++++++++ ++ ++ ++++++++ if (rem > *available)
++++++++++++ ++ ++ ++++++++ *available = rem;
++++++++++++ ++ ++ ++++++++ if (!run && num_bytes <= rem) {
++++++++++++ ++ ++ ++++++++ run = r;
++++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (run) {
++++++++++++ ++ ++ ++++++++ *ret_offset = run->offset;
++++++++++++ ++ ++ ++++++++ run_live_set(run, run->offset, num_bytes, true);
++++++++++++ ++ ++ ++++++++ run->offset += num_bytes;
++++++++++++ ++ ++ ++++++++ run->inflight_bytes += num_bytes;
++++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
++++++++++++ ++ ++ ++++++++ if (run->offset == run->end)
++++++++++++ ++ ++ ++++++++ run->open = false; /* full: closed, no tail */
++++++++++++ ++ ++ ++++++++ else
++++++++++++ ++ ++ ++++++++ tail_len = stripe_run_place(bg, run, &tail_start);
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (tail_len)
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
++++++++++++ ++ ++ ++++++++ for (int i = 0; i < nr_evict; i++)
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, evict_start[i], evict_len[i]);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ for (int i = 0; i < nr_evict; i++)
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, evict_start[i], evict_len[i]);
++++++++++++ ++ ++ ++++++++ if (nr_evict)
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Zeroed, so the error paths can free ->live and ->written. */
++++++++++++ ++ ++ ++++++++ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
++++++++++++ ++ ++ ++++++++ if (!new_run)
++++++++++++ ++ ++ ++++++++ return -ENOMEM;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ ret = btrfs_claim_free_stripe_run(bg,
++++++++++++ ++ ++ ++++++++ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
++++++++++++ ++ ++ ++++++++ &start, &len);
++++++++++++ ++ ++ ++++++++ if (ret)
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ if (len < num_bytes) {
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Only a shorter run is free. Put it back untouched (it is
++++++++++++ ++ ++ ++++++++ * still fully-free stripes) and let the caller retry with a
++++++++++++ ++ ++ ++++++++ * smaller allocation.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
++++++++++++ ++ ++ ++++++++ *available = max(*available, len);
++++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Sleeps, so it has to happen before the locks below. */
++++++++++++ ++ ++ ++++++++ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
++++++++++++ ++ ++ ++++++++ new_run->written = new_run->live_nbits ?
++++++++++++ ++ ++ ++++++++ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Install under fs_info->open_stripe_lock so that the block group
++++++++++++ ++ ++ ++++++++ * membership, the retire sequence read and the run installation are
++++++++++++ ++ ++ ++++++++ * one atomic step against the commit's retire walk (which bumps the
++++++++++++ ++ ++ ++++++++ * sequence and snapshots the membership list under the same lock).
++++++++++++ ++ ++ ++++++++ * A run is therefore either visible to the walk that retires its
++++++++++++ ++ ++ ++++++++ * window, or stamped with a sequence that walk's transaction did
++++++++++++ ++ ++ ++++++++ * not retire, which defers its extents to the next transaction.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ if (list_empty(&bg->open_stripe_bg_list)) {
++++++++++++ ++ ++ ++++++++ btrfs_get_block_group(bg);
++++++++++++ ++ ++ ++++++++ list_add_tail(&bg->open_stripe_bg_list,
++++++++++++ ++ ++ ++++++++ &fs_info->open_stripe_bgs);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (bg->ro) {
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Lost a race with this block group turning read-only; run
++++++++++++ ++ ++ ++++++++ * retirement for it may already have happened, so a new run
++++++++++++ ++ ++ ++++++++ * must not be installed.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
++++++++++++ ++ ++ ++++++++ *available = 0;
++++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Growable frontier: if the claimed region directly follows an open run
++++++++++++ ++ ++ ++++++++ * of the same class, grow that run over it and place this allocation at
++++++++++++ ++ ++ ++++++++ * the run's old tail, so the tail is used rather than stranded when the
++++++++++++ ++ ++ ++++++++ * run closes. Stripe exclusivity holds: the grown stripes were claimed
++++++++++++ ++ ++ ++++++++ * fully free, and the run is still retired at commit. The run keeps
++++++++++++ ++ ++ ++++++++ * its open_seq: it can only still be in a band slot if the retire walk
++++++++++++ ++ ++ ++++++++ * for its window has not run, which this lock nesting excludes.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) {
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *r = bg->open_stripe[class][band];
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!r || r->end != start)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /* Never grow another transaction's run (see the scan above). */
++++++++++++ ++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_META && r->transid != transid)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Only grow as far as the run's liveness map reaches; it
++++++++++++ ++ ++ ++++++++ * cannot be reallocated here (see alloc_run_live_map()).
++++++++++++ ++ ++ ++++++++ * Beyond that, leave the run alone and open a new one.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (((start + len - r->start) >> fs_info->sectorsize_bits) >
++++++++++++ ++ ++ ++++++++ r->live_nbits)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ ASSERT(r->open);
++++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
++++++++++++ ++ ++ ++++++++ *ret_offset = r->offset;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Mark the handed-out bytes live before anything can consult
++++++++++++ ++ ++ ++++++++ * the map: this allocation begins at the old tail, usually
++++++++++++ ++ ++ ++++++++ * mid-stripe, and a sector the map calls dead is a sector a
++++++++++++ ++ ++ ++++++++ * neighbour's padded write may zero-fill.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ run_live_set(r, r->offset, num_bytes, true);
++++++++++++ ++ ++ ++++++++ r->end = start + len;
++++++++++++ ++ ++ ++++++++ r->offset += num_bytes;
++++++++++++ ++ ++ ++++++++ r->inflight_bytes += num_bytes;
++++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder += len;
++++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
++++++++++++ ++ ++ ++++++++ if (r->offset == r->end)
++++++++++++ ++ ++ ++++++++ r->open = false; /* full: closed, no tail */
++++++++++++ ++ ++ ++++++++ else
++++++++++++ ++ ++ ++++++++ tail_len = stripe_run_place(bg, r, &tail_start);
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ if (tail_len)
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ ret = 0;
++++++++++++ ++ ++ ++++++++ goto out; /* frees the unused new_run */
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ new_run->bg = bg;
++++++++++++ ++ ++ ++++++++ new_run->class = class;
++++++++++++ ++ ++ ++++++++ new_run->persistent = false;
++++++++++++ ++ ++ ++++++++ new_run->retained = class == BTRFS_STRIPE_RUN_RELOC;
++++++++++++ ++ ++ ++++++++ if (new_run->retained)
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.reloc_run_retained);
++++++++++++ ++ ++ ++++++++ new_run->owner = 0;
++++++++++++ ++ ++ ++++++++ new_run->transid = transid;
++++++++++++ ++ ++ ++++++++ new_run->start = start;
++++++++++++ ++ ++ ++++++++ new_run->end = start + len;
++++++++++++ ++ ++ ++++++++ new_run->offset = start + num_bytes;
++++++++++++ ++ ++ ++++++++ new_run->inflight_bytes = num_bytes;
++++++++++++ ++ ++ ++++++++ new_run->open_seq = fs_info->stripe_retire_seq;
++++++++++++ ++ ++ ++++++++ new_run->open = (new_run->offset != new_run->end);
++++++++++++ ++ ++ ++++++++ run_live_set(new_run, start, num_bytes, true);
++++++++++++ ++ ++ ++++++++ list_add_tail(&new_run->list, &bg->open_stripe_runs);
++++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder += new_run->end - new_run->offset;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Place the new run in its remaining-space band, closing whichever run
++++++++++++ ++ ++ ++++++++ * loses a band collision (the last, fuller opener wins). Other bands'
++++++++++++ ++ ++ ++++++++ * open runs are untouched.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (new_run->open)
++++++++++++ ++ ++ ++++++++ tail_len = stripe_run_place(bg, new_run, &tail_start);
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (tail_len)
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ *ret_offset = start;
++++++++++++ ++ ++ ++++++++ new_run = NULL;
++++++++++++ ++ ++ ++++++++ ret = 0;
++++++++++++ ++ ++ ++++++++out:
++++++++++++ ++ ++ ++++++++ /* Unused on every path that did not install it, including errors. */
++++++++++++ ++ ++ ++++++++ if (new_run) {
++++++++++++ ++ ++ ++++++++ bitmap_free(new_run->live);
++++++++++++ ++ ++ ++++++++ bitmap_free(new_run->written);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ kfree(new_run);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Allocate a log tree block from the private run of this log commit.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * The run is keyed by @owner (the subvolume's root id) and @log_transid, so
++++++++++++ ++ ++ ++++++++ * each log commit gets whole stripes of its own: its blocks are durable once
++++++++++++ ++ ++ ++++++++ * fsync returns, and no later writer may share a stripe with them (see
++++++++++++ ++ ++ ++++++++ * BTRFS_STRIPE_RUN_META_LOG). An open run of the same owner from an earlier
++++++++++++ ++ ++ ++++++++ * log commit is closed, as a stale-transid metadata run is in
++++++++++++ ++ ++ ++++++++ * btrfs_alloc_from_open_stripe(). Like the other private runs, it is never
++++++++++++ ++ ++ ++++++++ * placed in a band slot.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in this
++++++++++++ ++ ++ ++++++++ * block group can satisfy the allocation, or -ENOMEM.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++int btrfs_alloc_from_log_stripe_run(struct btrfs_block_group *bg, u64 num_bytes,
++++++++++++ ++ ++ ++++++++ u64 owner, u64 log_transid,
++++++++++++ ++ ++ ++++++++ u64 *ret_offset, u64 *available)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
++++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
++++++++++++ ++ ++ ++++++++ const enum btrfs_stripe_run_class class = BTRFS_STRIPE_RUN_META_LOG;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *new_run;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ u64 tail_start = 0;
++++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
++++++++++++ ++ ++ ++++++++ u64 start;
++++++++++++ ++ ++ ++++++++ u64 len;
++++++++++++ ++ ++ ++++++++ int ret;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ ASSERT(num_bytes);
++++++++++++ ++ ++ ++++++++ *available = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (run->class != class || run->owner != owner || !run->open)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * A previous log commit of the same subvolume. Its blocks are
++++++++++++ ++ ++ ++++++++ * durable, so this one must not continue into them; close it
++++++++++++ ++ ++ ++++++++ * and claim fresh stripes below. log_transid starts over with
++++++++++++ ++ ++ ++++++++ * every transaction's log tree, so a run from an earlier
++++++++++++ ++ ++ ++++++++ * transaction can carry the same log_transid: the transid
++++++++++++ ++ ++ ++++++++ * tells them apart. Joining it would hand that transaction's
++++++++++++ ++ ++ ++++++++ * commit, which retires the run and waits for its blocks, a
++++++++++++ ++ ++ ++++++++ * block only this transaction will write.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (run->log_transid != log_transid ||
++++++++++++ ++ ++ ++++++++ run->transid != btrfs_get_fs_generation(fs_info)) {
++++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (num_bytes <= run->end - run->offset) {
++++++++++++ ++ ++ ++++++++ *ret_offset = run->offset;
++++++++++++ ++ ++ ++++++++ run_live_set(run, run->offset, num_bytes, true);
++++++++++++ ++ ++ ++++++++ run->offset += num_bytes;
++++++++++++ ++ ++ ++++++++ run->inflight_bytes += num_bytes;
++++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
++++++++++++ ++ ++ ++++++++ if (run->offset == run->end)
++++++++++++ ++ ++ ++++++++ run->open = false; /* full: closed, no tail */
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ /* Too small for this block: replace it. */
++++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (tail_len) {
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Zeroed: the error paths below free ->live and must find NULL. */
++++++++++++ ++ ++ ++++++++ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
++++++++++++ ++ ++ ++++++++ if (!new_run)
++++++++++++ ++ ++ ++++++++ return -ENOMEM;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ ret = btrfs_claim_free_stripe_run(bg,
++++++++++++ ++ ++ ++++++++ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
++++++++++++ ++ ++ ++++++++ &start, &len);
++++++++++++ ++ ++ ++++++++ if (ret)
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ if (len < num_bytes) {
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
++++++++++++ ++ ++ ++++++++ *available = max(*available, len);
++++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Sleeps, so it has to happen before the locks below. */
++++++++++++ ++ ++ ++++++++ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
++++++++++++ ++ ++ ++++++++ new_run->written = new_run->live_nbits ?
++++++++++++ ++ ++ ++++++++ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ if (list_empty(&bg->open_stripe_bg_list)) {
++++++++++++ ++ ++ ++++++++ btrfs_get_block_group(bg);
++++++++++++ ++ ++ ++++++++ list_add_tail(&bg->open_stripe_bg_list,
++++++++++++ ++ ++ ++++++++ &fs_info->open_stripe_bgs);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (bg->ro) {
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
++++++++++++ ++ ++ ++++++++ *available = 0;
++++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ new_run->bg = bg;
++++++++++++ ++ ++ ++++++++ new_run->class = class;
++++++++++++ ++ ++ ++++++++ new_run->persistent = false;
++++++++++++ ++ ++ ++++++++ new_run->root = owner;
++++++++++++ ++ ++ ++++++++ new_run->owner = owner;
++++++++++++ ++ ++ ++++++++ new_run->log_transid = log_transid;
++++++++++++ ++ ++ ++++++++ new_run->transid = btrfs_get_fs_generation(fs_info);
++++++++++++ ++ ++ ++++++++ new_run->start = start;
++++++++++++ ++ ++ ++++++++ new_run->end = start + len;
++++++++++++ ++ ++ ++++++++ new_run->offset = start + num_bytes;
++++++++++++ ++ ++ ++++++++ new_run->inflight_bytes = num_bytes;
++++++++++++ ++ ++ ++++++++ new_run->open_seq = fs_info->stripe_retire_seq;
++++++++++++ ++ ++ ++++++++ new_run->open = (new_run->offset != new_run->end);
++++++++++++ ++ ++ ++++++++ run_live_set(new_run, start, num_bytes, true);
++++++++++++ ++ ++ ++++++++ list_add_tail(&new_run->list, &bg->open_stripe_runs);
++++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder += new_run->end - new_run->offset;
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.log_run_opened);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ *ret_offset = start;
++++++++++++ ++ ++ ++++++++ new_run = NULL;
++++++++++++ ++ ++ ++++++++ ret = 0;
++++++++++++ ++ ++ ++++++++out:
++++++++++++ ++ ++ ++++++++ if (new_run) {
++++++++++++ ++ ++ ++++++++ bitmap_free(new_run->live);
++++++++++++ ++ ++ ++++++++ bitmap_free(new_run->written);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ kfree(new_run);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * The owner a private run is found by: the inode, or with
++++++++++++ ++ ++ ++++++++ * stripe_log_data_shared a LOG-class run shared by every inode of the
++++++++++++ ++ ++ ++++++++ * subvolume logging in the same log transaction. An allocation of a full
++++++++++++ ++ ++ ++++++++ * stripe or more keeps the inode's private run, so small fsync extents never
++++++++++++ ++ ++ ++++++++ * share the partial last stripe of a large one and wait for its IO. Bit 63
++++++++++++ ++ ++ ++++++++ * keeps the shared keys apart from inode numbers; the subvolume is matched
++++++++++++ ++ ++ ++++++++ * separately, by run->root.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static u64 stripe_run_owner(struct btrfs_inode *inode,
++++++++++++ ++ ++ ++++++++ enum btrfs_stripe_run_class class, u64 num_bytes,
++++++++++++ ++ ++ ++++++++ u64 fsl)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_root *root = inode->root;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_LOG && num_bytes < fsl &&
++++++++++++ ++ ++ ++++++++ READ_ONCE(root->fs_info->stripe_log_data_shared))
++++++++++++ ++ ++ ++++++++ return BIT_ULL(63) | (u32)READ_ONCE(root->log_transid);
++++++++++++ ++ ++ ++++++++ return btrfs_ino(inode);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Allocate num_bytes for @inode from its private run of @class (LOG or
++++++++++++ ++ ++ ++++++++ * NOCOW) in this block group, claiming a fresh run when there is none or it
++++++++++++ ++ ++ ++++++++ * does not fit. With stripe_log_data_shared, small LOG allocations use the
++++++++++++ ++ ++ ++++++++ * run the subvolume shares for the current log transaction
++++++++++++ ++ ++ ++++++++ * (stripe_run_owner()). Private runs are never placed in a band slot and
++++++++++++ ++ ++ ++++++++ * are found by subvolume and owner, so a log commit settles only stripes
++++++++++++ ++ ++ ++++++++ * holding its own extents. A NOCOW allocation may instead re-adopt the
++++++++++++ ++ ++ ++++++++ * free tail of the inode's last partial stripe (@readopt_hint).
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * A run too small for this allocation is closed at once rather than left
++++++++++++ ++ ++ ++++++++ * to strand its tail until commit. Two racing allocations for one owner
++++++++++++ ++ ++ ++++++++ * may each claim a run; the spare drains at the next commit, costing a
++++++++++++ ++ ++ ++++++++ * trapped tail, not correctness.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in
++++++++++++ ++ ++ ++++++++ * this block group can satisfy the allocation, or -ENOMEM.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++int btrfs_alloc_from_inode_stripe_run(struct btrfs_block_group *bg,
++++++++++++ ++ ++ ++++++++ struct btrfs_inode *inode,
++++++++++++ ++ ++ ++++++++ enum btrfs_stripe_run_class class,
++++++++++++ ++ ++ ++++++++ u64 num_bytes, u64 readopt_hint,
++++++++++++ ++ ++ ++++++++ u64 *ret_offset, u64 *available)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
++++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
++++++++++++ ++ ++ ++++++++ const u64 root_id = btrfs_root_id(inode->root);
++++++++++++ ++ ++ ++++++++ const u64 ino = stripe_run_owner(inode, class, num_bytes, fsl);
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *new_run;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ u64 tail_start = 0;
++++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
++++++++++++ ++ ++ ++++++++ u64 start;
++++++++++++ ++ ++ ++++++++ u64 len;
++++++++++++ ++ ++ ++++++++ int ret;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ ASSERT(num_bytes);
++++++++++++ ++ ++ ++++++++ *available = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (run->class != class || run->root != root_id ||
++++++++++++ ++ ++ ++++++++ run->owner != ino || !run->open)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (num_bytes <= run->end - run->offset) {
++++++++++++ ++ ++ ++++++++ *ret_offset = run->offset;
++++++++++++ ++ ++ ++++++++ run_live_set(run, run->offset, num_bytes, true);
++++++++++++ ++ ++ ++++++++ run->offset += num_bytes;
++++++++++++ ++ ++ ++++++++ run->inflight_bytes += num_bytes;
++++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
++++++++++++ ++ ++ ++++++++ if (run->offset == run->end)
++++++++++++ ++ ++ ++++++++ run->open = false; /* full: closed, no tail */
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (tail_len) {
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Zeroed, so the error paths can free ->live and ->written. */
++++++++++++ ++ ++ ++++++++ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
++++++++++++ ++ ++ ++++++++ if (!new_run)
++++++++++++ ++ ++ ++++++++ return -ENOMEM;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Re-adoption (nocow only): after a remount (or eviction) the
++++++++++++ ++ ++ ++++++++ * inode's last partial stripe has no run, and a fresh claim would
++++++++++++ ++ ++ ++++++++ * abandon its free tail forever. When the allocation hint points
++++++++++++ ++ ++ ++++++++ * into a stripe wholly owned by this inode, adopt the tail as the
++++++++++++ ++ ++ ++++++++ * private run instead: every byte below the frontier is provably
++++++++++++ ++ ++ ++++++++ * the inode's own committed data, and nothing else can consume a
++++++++++++ ++ ++ ++++++++ * partial stripe's free space (see btrfs_claim_stripe_tail()).
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_NOCOW && readopt_hint &&
++++++++++++ ++ ++ ++++++++ readopt_hint >= bg->start &&
++++++++++++ ++ ++ ++++++++ readopt_hint < bg->start + bg->length) {
++++++++++++ ++ ++ ++++++++ u64 stripe = bg->start +
++++++++++++ ++ ++ ++++++++ stripe_round_down(readopt_hint - bg->start, fsl);
++++++++++++ ++ ++ ++++++++ u64 frontier = 0;
++++++++++++ ++ ++ ++++++++ u64 tlen;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Under space_info->groups_sem (we are inside
++++++++++++ ++ ++ ++++++++ * find_free_extent()), so the lookup must not block -- see
++++++++++++ ++ ++ ++++++++ * stripe_extents_owned_by(). A trylock failure just declines
++++++++++++ ++ ++ ++++++++ * the re-adoption and falls through to a fresh claim below,
++++++++++++ ++ ++ ++++++++ * costing this stripe's free tail, not correctness.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (stripe_extents_owned_by(fs_info, root_id, ino, stripe, fsl,
++++++++++++ ++ ++ ++++++++ &frontier, true) &&
++++++++++++ ++ ++ ++++++++ frontier > stripe && frontier < stripe + fsl &&
++++++++++++ ++ ++ ++++++++ num_bytes <= stripe + fsl - frontier) {
++++++++++++ ++ ++ ++++++++ tlen = stripe + fsl - frontier;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * btrfs_stripe_run_range_usable() is for whole-stripe
++++++++++++ ++ ++ ++++++++ * claims (it rejects anything shorter than a full
++++++++++++ ++ ++ ++++++++ * stripe); a tail needs only the run-overlap check.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (stripe_range_run_free(bg, frontier, tlen) &&
++++++++++++ ++ ++ ++++++++ !btrfs_claim_stripe_tail(bg, frontier, tlen)) {
++++++++++++ ++ ++ ++++++++ start = frontier;
++++++++++++ ++ ++ ++++++++ len = tlen;
++++++++++++ ++ ++ ++++++++ goto install;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ ret = btrfs_claim_free_stripe_run(bg,
++++++++++++ ++ ++ ++++++++ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
++++++++++++ ++ ++ ++++++++ &start, &len);
++++++++++++ ++ ++ ++++++++ if (ret)
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ if (len < num_bytes) {
++++++++++++ ++ ++ ++++++++ /* Only a shorter run is free; see btrfs_alloc_from_open_stripe(). */
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
++++++++++++ ++ ++ ++++++++ *available = len;
++++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++install:
++++++++++++ ++ ++ ++++++++ /* Sleeps, so it has to happen before the locks below. */
++++++++++++ ++ ++ ++++++++ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
++++++++++++ ++ ++ ++++++++ new_run->written = new_run->live_nbits ?
++++++++++++ ++ ++ ++++++++ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* See the installation comment in btrfs_alloc_from_open_stripe(). */
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ if (list_empty(&bg->open_stripe_bg_list)) {
++++++++++++ ++ ++ ++++++++ btrfs_get_block_group(bg);
++++++++++++ ++ ++ ++++++++ list_add_tail(&bg->open_stripe_bg_list,
++++++++++++ ++ ++ ++++++++ &fs_info->open_stripe_bgs);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (bg->ro) {
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
++++++++++++ ++ ++ ++++++++ *available = 0;
++++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ new_run->bg = bg;
++++++++++++ ++ ++ ++++++++ new_run->class = class;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * A NOCOW run outlives commits only while the stripe_alloc_allow_rmw
++++++++++++ ++ ++ ++++++++ * policy lets nocow data be written in place at all (see
++++++++++++ ++ ++ ++++++++ * close_block_group_stripe_runs()). Decided once, here: the commit
++++++++++++ ++ ++ ++++++++ * path's close and settle predicates must agree with each other for
++++++++++++ ++ ++ ++++++++ * the run's whole life, whatever the policy does meanwhile.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ new_run->persistent = class == BTRFS_STRIPE_RUN_NOCOW &&
++++++++++++ ++ ++ ++++++++ (btrfs_stripe_allow_rmw(fs_info) &
++++++++++++ ++ ++ ++++++++ (BTRFS_STRIPE_RMW_NODATACOW |
++++++++++++ ++ ++ ++++++++ BTRFS_STRIPE_RMW_PREALLOC));
++++++++++++ ++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_NOCOW)
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.nocow_run_opened);
++++++++++++ ++ ++ ++++++++ new_run->root = root_id;
++++++++++++ ++ ++ ++++++++ new_run->owner = ino;
++++++++++++ ++ ++ ++++++++ new_run->start = start;
++++++++++++ ++ ++ ++++++++ new_run->end = start + len;
++++++++++++ ++ ++ ++++++++ new_run->offset = start + num_bytes;
++++++++++++ ++ ++ ++++++++ new_run->inflight_bytes = num_bytes;
++++++++++++ ++ ++ ++++++++ new_run->open_seq = fs_info->stripe_retire_seq;
++++++++++++ ++ ++ ++++++++ new_run->open = (new_run->offset != new_run->end);
++++++++++++ ++ ++ ++++++++ run_live_set(new_run, start, num_bytes, true);
++++++++++++ ++ ++ ++++++++ list_add_tail(&new_run->list, &bg->open_stripe_runs);
++++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder += new_run->end - new_run->offset;
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ *ret_offset = start;
++++++++++++ ++ ++ ++++++++ new_run = NULL;
++++++++++++ ++ ++ ++++++++ ret = 0;
++++++++++++ ++ ++ ++++++++out:
++++++++++++ ++ ++ ++++++++ /* Unused on every path that did not install it, including errors. */
++++++++++++ ++ ++ ++++++++ if (new_run) {
++++++++++++ ++ ++ ++++++++ bitmap_free(new_run->live);
++++++++++++ ++ ++ ++++++++ bitmap_free(new_run->written);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ kfree(new_run);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * An extent inside a stripe run's range has been freed: its sectors no longer
++++++++++++ ++ ++ ++++++++ * hold data anyone intends to write, so clear their live bits.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * This is what makes a run detectably dead (open_stripe_run_dead()), so the
++++++++++++ ++ ++ ++++++++ * next commit can close it and release its stripes. Without it a nodatacow
++++++++++++ ++ ++ ++++++++ * or preallocating inode's private run hoards its stripes until the inode is
++++++++++++ ++ ++ ++++++++ * evicted. A partial write to one of those stripes can also pad over the
++++++++++++ ++ ++ ++++++++ * freed sectors now instead of falling back to a read-modify-write.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Takes only bg->stripe_run_lock, and the callers hold no free space lock
++++++++++++ ++ ++ ++++++++ * yet, so no nesting rule applies. The range is clamped to the run: a free
++++++++++++ ++ ++ ++++++++ * may span a run's boundary, and running off the live map would retire it.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_stripe_run_note_free(struct btrfs_block_group *bg, u64 bytenr,
++++++++++++ ++ ++ ++++++++ u64 size)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!btrfs_is_stripe_alloc_bg(bg))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ u64 start = max(bytenr, run->start);
++++++++++++ ++ ++ ++++++++ u64 end = min(bytenr + size, run->offset);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Only the persistent runs, which are the only ones that can
++++++++++++ ++ ++ ++++++++ * outlive their own data. Every other class is closed and
++++++++++++ ++ ++ ++++++++ * freed by the commit it belongs to, so its live map is never
++++++++++++ ++ ++ ++++++++ * consulted after its bytes could have been freed, and
++++++++++++ ++ ++ ++++++++ * clearing bits there would change padding decisions for no
++++++++++++ ++ ++ ++++++++ * gain here.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (!run->persistent)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (end <= start)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ run_live_set(run, start, end - start, false);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * A run's range must not be claimed again while the run object exists, even
++++++++++++ ++ ++ ++++++++ * if every byte of one of its stripes is back in the free space cache
++++++++++++ ++ ++ ++++++++ * (abandoned allocations): a second run over the same range would make the
++++++++++++ ++ ++ ++++++++ * range-to-run lookups (ordered extent attach, IO reporting) ambiguous and
++++++++++++ ++ ++ ++++++++ * corrupt the inflight accounting. Trim the candidate to end before the
++++++++++++ ++ ++ ++++++++ * first overlapping run, and reject it if its head overlaps or less than a
++++++++++++ ++ ++ ++++++++ * full stripe remains. Caller holds ctl->tree_lock; stripe_run_lock nests
++++++++++++ ++ ++ ++++++++ * inside it.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++bool btrfs_stripe_run_range_usable(struct btrfs_block_group *bg,
++++++++++++ ++ ++ ++++++++ u64 run_start, u64 *run_len)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ u64 len = *run_len;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (run->end <= run_start || run->start >= run_start + len)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (run->start <= run_start) {
++++++++++++ ++ ++ ++++++++ len = 0;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ len = run->start - run_start;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (len < bg->full_stripe_len) {
++++++++++++ ++ ++ ++++++++ atomic64_inc(&bg->fs_info->stripe_run_stats.claim_refused_live_run);
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ *run_len = len;
++++++++++++ ++ ++ ++++++++ return true;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * For the stripe_unusable scan: whole free stripes inside a live run's
++++++++++++ ++ ++ ++++++++ * range cannot be claimed while the run exists
++++++++++++ ++ ++ ++++++++ * (btrfs_stripe_run_range_usable()), so they must not count as claimable
++++++++++++ ++ ++ ++++++++ * or the admission gate would admit writes that cannot be placed. Clear
++++++++++++ ++ ++ ++++++++ * them from @freep and return their total; free_open_stripe_run() marks the
++++++++++++ ++ ++ ++++++++ * group for rescan so they are credited back when the run goes. Caller
++++++++++++ ++ ++ ++++++++ * holds ctl->tree_lock; stripe_run_lock nests inside it.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++u64 btrfs_stripe_run_stranded_stripes(struct btrfs_block_group *bg, u32 *freep,
++++++++++++ ++ ++ ++++++++ u64 nstripes)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ u64 stranded = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ u64 idx, last;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (run->end <= run->start)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ idx = div64_u64(run->start - bg->start, fsl);
++++++++++++ ++ ++ ++++++++ last = div64_u64(run->end - 1 - bg->start, fsl);
++++++++++++ ++ ++ ++++++++ for (; idx <= last && idx < nstripes; idx++) {
++++++++++++ ++ ++ ++++++++ if (freep[idx] != fsl)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ freep[idx] = 0;
++++++++++++ ++ ++ ++++++++ stranded += fsl;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (stranded)
++++++++++++ ++ ++ ++++++++ atomic64_add(stranded, &bg->fs_info->stripe_run_stats.strand_scan_bytes);
++++++++++++ ++ ++ ++++++++ return stranded;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Is the full stripe at @stripe inside a live run's range, i.e. stranded by
++++++++++++ ++ ++ ++++++++ * the rule of btrfs_stripe_run_stranded_stripes()? Caller holds
++++++++++++ ++ ++ ++++++++ * ctl->tree_lock.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++bool btrfs_stripe_run_strands(struct btrfs_block_group *bg, u64 stripe)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ bool ret = false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (run->end > run->start &&
++++++++++++ ++ ++ ++++++++ stripe < run->end && stripe + fsl > run->start) {
++++++++++++ ++ ++ ++++++++ ret = true;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Parse the value of the btrfs.stripe_alloc_allow_rmw property, a word list
++++++++++++ ++ ++ ++++++++ * ("nodatacow", "prealloc", "fsync"), into its mask. Words are separated by
++++++++++++ ++ ++ ++++++++ * comma, space or colon. An empty value yields an empty mask. Returns
++++++++++++ ++ ++ ++++++++ * -EINVAL on any unknown word.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++int btrfs_parse_stripe_rmw(const char *value, size_t len, u32 *mask)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ u32 m = 0;
++++++++++++ ++ ++ ++++++++ size_t i = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ while (i < len) {
++++++++++++ ++ ++ ++++++++ size_t j = i;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ while (j < len && value[j] != ',' && value[j] != ' ' &&
++++++++++++ ++ ++ ++++++++ value[j] != ':')
++++++++++++ ++ ++ ++++++++ j++;
++++++++++++ ++ ++ ++++++++ if (j - i == strlen("nodatacow") &&
++++++++++++ ++ ++ ++++++++ !strncmp(value + i, "nodatacow", j - i))
++++++++++++ ++ ++ ++++++++ m |= BTRFS_STRIPE_RMW_NODATACOW;
++++++++++++ ++ ++ ++++++++ else if (j - i == strlen("prealloc") &&
++++++++++++ ++ ++ ++++++++ !strncmp(value + i, "prealloc", j - i))
++++++++++++ ++ ++ ++++++++ m |= BTRFS_STRIPE_RMW_PREALLOC;
++++++++++++ ++ ++ ++++++++ else if (j - i == strlen("fsync") &&
++++++++++++ ++ ++ ++++++++ !strncmp(value + i, "fsync", j - i))
++++++++++++ ++ ++ ++++++++ m |= BTRFS_STRIPE_RMW_FSYNC;
++++++++++++ ++ ++ ++++++++ else if (j != i)
++++++++++++ ++ ++ ++++++++ return -EINVAL;
++++++++++++ ++ ++ ++++++++ i = j + 1;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ *mask = m;
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Does every committed extent in [start, start + len) belong solely to
++++++++++++ ++ ++ ++++++++ * @ino in @root_id -- exactly one reference, a plain data ref, count 1?
++++++++++++ ++ ++ ++++++++ * Any metadata block, shared reference, or foreign owner fails. Caller
++++++++++++ ++ ++ ++++++++ * limits the range to full stripes; uncommitted allocations need not be
++++++++++++ ++ ++ ++++++++ * visible because nothing can allocate into a partially used stripe, so a
++++++++++++ ++ ++ ++++++++ * foreign extent can only appear after the whole stripe frees and commits.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Callers holding space_info->groups_sem (anything reached from
++++++++++++ ++ ++ ++++++++ * find_free_extent()) must pass @nowait: delayed ref processing takes
++++++++++++ ++ ++ ++++++++ * extent tree locks and then groups_sem, so blocking on an extent tree lock
++++++++++++ ++ ++ ++++++++ * here can deadlock behind a queued groups_sem writer. Callers that must
++++++++++++ ++ ++ ++++++++ * not block for other reasons pass it too. With @nowait a contended lock
++++++++++++ ++ ++ ++++++++ * fails the search and the answer is false, which every caller treats only
++++++++++++ ++ ++ ++++++++ * as a reason to skip an optimisation.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static bool stripe_extents_owned_by(struct btrfs_fs_info *fs_info,
++++++++++++ ++ ++ ++++++++ u64 root_id, u64 ino, u64 start, u64 len,
++++++++++++ ++ ++ ++++++++ u64 *frontier, bool nowait)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_root *extent_root = btrfs_extent_root(fs_info, start);
++++++++++++ ++ ++ ++++++++ struct btrfs_path *path;
++++++++++++ ++ ++ ++++++++ struct btrfs_key key;
++++++++++++ ++ ++ ++++++++ bool ret = true;
++++++++++++ ++ ++ ++++++++ int err;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!extent_root)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ path = btrfs_alloc_path();
++++++++++++ ++ ++ ++++++++ if (!path)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Read-only search, so nowait is allowed, and every step of the walk
++++++++++++ ++ ++ ++++++++ * below honours it (btrfs_next_leaf() runs with time_seq 0).
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ path->nowait = nowait;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ key.objectid = start;
++++++++++++ ++ ++ ++++++++ key.type = BTRFS_EXTENT_ITEM_KEY;
++++++++++++ ++ ++ ++++++++ key.offset = 0;
++++++++++++ ++ ++ ++++++++ err = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
++++++++++++ ++ ++ ++++++++ if (err < 0) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ /* An extent starting before @start may still overlap it. */
++++++++++++ ++ ++ ++++++++ err = btrfs_previous_extent_item(extent_root, path, 0);
++++++++++++ ++ ++ ++++++++ if (err < 0) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (err > 0) {
++++++++++++ ++ ++ ++++++++ btrfs_release_path(path);
++++++++++++ ++ ++ ++++++++ key.objectid = start;
++++++++++++ ++ ++ ++++++++ key.type = BTRFS_EXTENT_ITEM_KEY;
++++++++++++ ++ ++ ++++++++ key.offset = 0;
++++++++++++ ++ ++ ++++++++ err = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
++++++++++++ ++ ++ ++++++++ if (err < 0) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ while (true) {
++++++++++++ ++ ++ ++++++++ struct extent_buffer *leaf = path->nodes[0];
++++++++++++ ++ ++ ++++++++ int slot = path->slots[0];
++++++++++++ ++ ++ ++++++++ struct btrfs_extent_item *ei;
++++++++++++ ++ ++ ++++++++ struct btrfs_extent_inline_ref *iref;
++++++++++++ ++ ++ ++++++++ struct btrfs_extent_data_ref *dref;
++++++++++++ ++ ++ ++++++++ int type;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (slot >= btrfs_header_nritems(leaf)) {
++++++++++++ ++ ++ ++++++++ err = btrfs_next_leaf(extent_root, path);
++++++++++++ ++ ++ ++++++++ if (err < 0) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (err > 0)
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ btrfs_item_key_to_cpu(leaf, &key, slot);
++++++++++++ ++ ++ ++++++++ if (key.objectid >= start + len)
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ if (key.type == BTRFS_METADATA_ITEM_KEY) {
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * key.offset is the level, not a length; a tree
++++++++++++ ++ ++ ++++++++ * block spans nodesize bytes. One from before the
++++++++++++ ++ ++ ++++++++ * range (e.g. where the pre-range positioning landed
++++++++++++ ++ ++ ++++++++ * on a metadata block group's last item) is harmless.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (key.objectid + fs_info->nodesize > start) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ path->slots[0]++;
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (key.type != BTRFS_EXTENT_ITEM_KEY ||
++++++++++++ ++ ++ ++++++++ key.objectid + key.offset <= start) {
++++++++++++ ++ ++ ++++++++ path->slots[0]++;
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item);
++++++++++++ ++ ++ ++++++++ if (btrfs_extent_refs(leaf, ei) != 1 ||
++++++++++++ ++ ++ ++++++++ !(btrfs_extent_flags(leaf, ei) & BTRFS_EXTENT_FLAG_DATA)) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * With a keyed (non-inline) backref the item ends after the
++++++++++++ ++ ++ ++++++++ * header, and reading an inline ref would read the next item.
++++++++++++ ++ ++ ++++++++ * Treat it as not owned, the conservative answer.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (btrfs_item_size(leaf, slot) < sizeof(*ei) +
++++++++++++ ++ ++ ++++++++ btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY)) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ iref = (struct btrfs_extent_inline_ref *)(ei + 1);
++++++++++++ ++ ++ ++++++++ type = btrfs_get_extent_inline_ref_type(leaf, iref,
++++++++++++ ++ ++ ++++++++ BTRFS_REF_TYPE_DATA);
++++++++++++ ++ ++ ++++++++ if (type != BTRFS_EXTENT_DATA_REF_KEY) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ dref = (struct btrfs_extent_data_ref *)(&iref->offset);
++++++++++++ ++ ++ ++++++++ if (btrfs_extent_data_ref_root(leaf, dref) != root_id ||
++++++++++++ ++ ++ ++++++++ btrfs_extent_data_ref_objectid(leaf, dref) != ino ||
++++++++++++ ++ ++ ++++++++ btrfs_extent_data_ref_count(leaf, dref) != 1) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (frontier && key.objectid + key.offset > *frontier)
++++++++++++ ++ ++ ++++++++ *frontier = min(key.objectid + key.offset, start + len);
++++++++++++ ++ ++ ++++++++ path->slots[0]++;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++out:
++++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * May [start, start + num_bytes) be written in place under stripe-exclusive
++++++++++++ ++ ++ ++++++++ * allocation? Only when stripe_alloc_allow_rmw includes @which and every
++++++++++++ ++ ++ ++++++++ * full stripe the range touches is isolated to @inode: an in-place write
++++++++++++ ++ ++ ++++++++ * RMWs parity, so a crash can tear every extent sharing the stripe, and
++++++++++++ ++ ++ ++++++++ * isolation confines that to the file that opted out of COW. Fast path:
++++++++++++ ++ ++ ++++++++ * the stripe is in one of the inode's own NOCOW runs. Slow path: the
++++++++++++ ++ ++ ++++++++ * committed extent tree proves sole ownership. Extents that fail stay
++++++++++++ ++ ++ ++++++++ * COWed, and their rewrite lands in the inode's NOCOW run, so the next
++++++++++++ ++ ++ ++++++++ * overwrite passes.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++bool btrfs_stripe_nocow_writable(struct btrfs_inode *inode, u64 start,
++++++++++++ ++ ++ ++++++++ u64 num_bytes, u32 which, bool nowait)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = inode->root->fs_info;
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ const u64 root_id = btrfs_root_id(inode->root);
++++++++++++ ++ ++ ++++++++ const u64 ino = btrfs_ino(inode);
++++++++++++ ++ ++ ++++++++ u64 fsl;
++++++++++++ ++ ++ ++++++++ u64 cur;
++++++++++++ ++ ++ ++++++++ u64 end;
++++++++++++ ++ ++ ++++++++ bool ret = true;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!(btrfs_stripe_allow_rmw(fs_info) & which))
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, start);
++++++++++++ ++ ++ ++++++++ if (!bg)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ if (!btrfs_is_stripe_alloc_bg(bg)) {
++++++++++++ ++ ++ ++++++++ /* Not a raid56 data group: nothing forces COW here anyway. */
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ return true;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ fsl = bg->full_stripe_len;
++++++++++++ ++ ++ ++++++++ cur = bg->start + stripe_round_down(start - bg->start, fsl);
++++++++++++ ++ ++ ++++++++ end = start + num_bytes;
++++++++++++ ++ ++ ++++++++ while (cur < end) {
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ int verdict = 0; /* 0 = unknown, 1 = ok, -1 = foreign */
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (cur < run->start || cur >= run->end)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (run->class == BTRFS_STRIPE_RUN_NOCOW &&
++++++++++++ ++ ++ ++++++++ run->root == root_id && run->owner == ino)
++++++++++++ ++ ++ ++++++++ verdict = 1;
++++++++++++ ++ ++ ++++++++ else
++++++++++++ ++ ++ ++++++++ verdict = -1;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * No groups_sem here (this runs from the nocow check, not
++++++++++++ ++ ++ ++++++++ * from the allocator), so the reliable blocking search is
++++++++++++ ++ ++ ++++++++ * both safe and wanted: a spurious "not ours" would force
++++++++++++ ++ ++ ++++++++ * COW on an extent that does not need it. A NOWAIT write
++++++++++++ ++ ++ ++++++++ * must not block, so it gets the nonblocking search, and
++++++++++++ ++ ++ ++++++++ * COW (which it cannot do either) when that cannot answer.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (verdict == 0 &&
++++++++++++ ++ ++ ++++++++ !stripe_extents_owned_by(fs_info, root_id, ino, cur, fsl,
++++++++++++ ++ ++ ++++++++ NULL, nowait))
++++++++++++ ++ ++ ++++++++ verdict = -1;
++++++++++++ ++ ++ ++++++++ if (verdict < 0) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ cur += fsl;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Lower an allocation ceiling to the end of the last live sector at or above
++++++++++++ ++ ++ ++++++++ * @floor. Sectors between it and the frontier were allocated and abandoned:
++++++++++++ ++ ++ ++++++++ * they never get a bio, so a parked rbio counting on them would leave only
++++++++++++ ++ ++ ++++++++ * when kicked or at its park deadline. Caller holds bg->stripe_run_lock.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static u64 run_live_ceiling(const struct btrfs_open_stripe_run *run, u64 floor,
++++++++++++ ++ ++ ++++++++ u64 ceiling)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
++++++++++++ ++ ++ ++++++++ u32 first;
++++++++++++ ++ ++ ++++++++ u32 i;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!run->live || !run->live_nbits || floor < run->start ||
++++++++++++ ++ ++ ++++++++ ceiling <= floor)
++++++++++++ ++ ++ ++++++++ return ceiling;
++++++++++++ ++ ++ ++++++++ first = (floor - run->start) >> bits;
++++++++++++ ++ ++ ++++++++ i = (ceiling - run->start) >> bits;
++++++++++++ ++ ++ ++++++++ if (i > run->live_nbits)
++++++++++++ ++ ++ ++++++++ return ceiling;
++++++++++++ ++ ++ ++++++++ while (i > first && !test_bit(i - 1, run->live))
++++++++++++ ++ ++ ++++++++ i--;
++++++++++++ ++ ++ ++++++++ return run->start + ((u64)i << bits);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * How far has the covering run allocated into this stripe? Returns the
++++++++++++ ++ ++ ++++++++ * exclusive end of the allocated prefix, clamped to the stripe, or U64_MAX
++++++++++++ ++ ++ ++++++++ * when no run covers the stripe (nothing left to wait for; the caller
++++++++++++ ++ ++ ++++++++ * proceeds as if the stripe were fully allocated and fully arrived).
++++++++++++ ++ ++ ++++++++ * A partial write below the returned ceiling is waiting on neighbours whose
++++++++++++ ++ ++ ++++++++ * data IO is already in flight -- allocation happens at writeback submission
++++++++++++ ++ ++ ++++++++ * and the run's inflight accounting makes the commit wait for it -- so a
++++++++++++ ++ ++ ++++++++ * parked rbio covering [stripe_start, ceiling) can complete (padding the
++++++++++++ ++ ++ ++++++++ * dead tail), and one that does not should keep waiting for merges.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++u64 btrfs_stripe_run_alloc_ceiling(struct btrfs_fs_info *fs_info,
++++++++++++ ++ ++ ++++++++ u64 stripe_start, u64 stripe_len)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ u64 ceiling = U64_MAX;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
++++++++++++ ++ ++ ++++++++ return U64_MAX;
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
++++++++++++ ++ ++ ++++++++ if (!bg)
++++++++++++ ++ ++ ++++++++ return U64_MAX;
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (stripe_start < run->start || stripe_start >= run->end)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ ceiling = clamp(run->offset, stripe_start,
++++++++++++ ++ ++ ++++++++ stripe_start + stripe_len);
++++++++++++ ++ ++ ++++++++ ceiling = run_live_ceiling(run, stripe_start, ceiling);
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ return ceiling;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Which sectors of this stripe may be padded?
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Fills @dead with one bit per sector, set where the sector holds nothing
++++++++++++ ++ ++ ++++++++ * anyone has written: never handed out, handed out and abandoned, or
++++++++++++ ++ ++ ++++++++ * handed out with the write not yet completed (per the run's written map).
++++++++++++ ++ ++ ++++++++ * The caller may zero-fill exactly those and write the whole stripe.
++++++++++++ ++ ++ ++++++++ * @unwritten, when non-NULL, gets the subset that is allocated but
++++++++++++ ++ ++ ++++++++ * unwritten.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Zeroing an allocated sector whose write is still to come is safe: the
++++++++++++ ++ ++ ++++++++ * caller holds the stripe lock, which every raid56 write to the stripe
++++++++++++ ++ ++ ++++++++ * takes, so that write lands after the caller's; completed writes are in
++++++++++++ ++ ++ ++++++++ * the written map before the stripe is unlocked (rbio_orig_end_io()); and
++++++++++++ ++ ++ ++++++++ * nothing reads the sector before its own write lands, since its extent is
++++++++++++ ++ ++ ++++++++ * not committed until then. A NOCOW run's written map is not consulted:
++++++++++++ ++ ++ ++++++++ * its live sectors are rewritten in place, so they may already hold data.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Returns false if no run covers the stripe; nothing may be padded then,
++++++++++++ ++ ++ ++++++++ * since an uncovered sector may hold committed data. @info, when non-NULL,
++++++++++++ ++ ++ ++++++++ * records the covering run's state for the pad-decline trace.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++bool btrfs_stripe_run_pad_mask(struct btrfs_fs_info *fs_info, u64 stripe_start,
++++++++++++ ++ ++ ++++++++ u64 stripe_len, unsigned long *dead,
++++++++++++ ++ ++ ++++++++ unsigned long *unwritten,
++++++++++++ ++ ++ ++++++++ struct btrfs_stripe_pad_info *info)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ const u32 bits = fs_info->sectorsize_bits;
++++++++++++ ++ ++ ++++++++ const u32 nsectors = stripe_len >> bits;
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ bool ret = false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (info) {
++++++++++++ ++ ++ ++++++++ memset(info, 0, sizeof(*info));
++++++++++++ ++ ++ ++++++++ info->reason = BTRFS_STRIPE_PAD_NO_RUN;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs)) {
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
++++++++++++ ++ ++ ++++++++ if (info)
++++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_BGS;
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
++++++++++++ ++ ++ ++++++++ if (!bg) {
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
++++++++++++ ++ ++ ++++++++ if (info)
++++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_BG;
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (info)
++++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_COVER;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ u32 first;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (stripe_start < run->start || stripe_start >= run->end)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (info) {
++++++++++++ ++ ++ ++++++++ info->run_start = run->start;
++++++++++++ ++ ++ ++++++++ info->run_end = run->end;
++++++++++++ ++ ++ ++++++++ info->run_offset = run->offset;
++++++++++++ ++ ++ ++++++++ info->inflight_bytes = run->inflight_bytes;
++++++++++++ ++ ++ ++++++++ info->run_open = run->open;
++++++++++++ ++ ++ ++++++++ info->run_class = run->class;
++++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_LIVE;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (!run->live || !run->live_nbits)
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * The stripe must lie inside the run's span. A closed run
++++++++++++ ++ ++ ++++++++ * has shrunk its end to the frontier, so its final stripe
++++++++++++ ++ ++ ++++++++ * overhangs: the sectors past end are the returned tail,
++++++++++++ ++ ++ ++++++++ * sub-stripe free space nothing can claim while the stripe
++++++++++++ ++ ++ ++++++++ * holds live data. They were never allocated, their live
++++++++++++ ++ ++ ++++++++ * bits are clear, and they are exactly what a kicked park
++++++++++++ ++ ++ ++++++++ * needs to pad. An open run's end is stripe aligned, so
++++++++++++ ++ ++ ++++++++ * overhang there means a stripe the run does not own.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (stripe_start + stripe_len > run->end && run->open) {
++++++++++++ ++ ++ ++++++++ if (info)
++++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_OVERHANG;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ first = (stripe_start - run->start) >> bits;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * And wholly inside the map. Declining here costs a
++++++++++++ ++ ++ ++++++++ * read-modify-write; reading past the map would be a lie about
++++++++++++ ++ ++ ++++++++ * which sectors are dead, and padding acts on that by
++++++++++++ ++ ++ ++++++++ * overwriting them.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (first + nsectors > run->live_nbits) {
++++++++++++ ++ ++ ++++++++ if (info)
++++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_PAST_MAP;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ bitmap_zero(dead, nsectors);
++++++++++++ ++ ++ ++++++++ if (unwritten)
++++++++++++ ++ ++ ++++++++ bitmap_zero(unwritten, nsectors);
++++++++++++ ++ ++ ++++++++ for (u32 i = 0; i < nsectors; i++) {
++++++++++++ ++ ++ ++++++++ if (!test_bit(first + i, run->live)) {
++++++++++++ ++ ++ ++++++++ set_bit(i, dead);
++++++++++++ ++ ++ ++++++++ } else if (run->written &&
++++++++++++ ++ ++ ++++++++ run->class != BTRFS_STRIPE_RUN_NOCOW &&
++++++++++++ ++ ++ ++++++++ !test_bit(first + i, run->written)) {
++++++++++++ ++ ++ ++++++++ set_bit(i, dead);
++++++++++++ ++ ++ ++++++++ if (unwritten)
++++++++++++ ++ ++ ++++++++ set_bit(i, unwritten);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (info)
++++++++++++ ++ ++ ++++++++ info->reason = BTRFS_STRIPE_PAD_OK;
++++++++++++ ++ ++ ++++++++ ret = true;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (!ret)
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/* No open or draining run covers @bytenr any more. */
++++++++++++ ++ ++ ++++++++static bool stripe_log_range_settled(struct btrfs_block_group *bg, u64 bytenr)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ bool settled = true;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (bytenr >= run->start && bytenr < run->offset) {
++++++++++++ ++ ++ ++++++++ settled = false;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ return settled;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Remember, on the logging inode, a logged extent that lives in the final
++++++++++++ ++ ++ ++++++++ * partial stripe of one of the inode's recently closed runs, so that the
++++++++++++ ++ ++ ++++++++ * next fsync can re-dirty its file range and carry the data forward into
++++++++++++ ++ ++ ++++++++ * the then-current run -- letting the old partial stripe free whole. A
++++++++++++ ++ ++ ++++++++ * zero @file_len means the caller does not want this extent carried
++++++++++++ ++ ++ ++++++++ * (e.g. compressed extents, which would recompress). @ps_len is nonzero
++++++++++++ ++ ++ ++++++++ * when the settling walk could see the run and computed its final partial
++++++++++++ ++ ++ ++++++++ * stripe. Every table here is small and bounded; on overflow the extent
++++++++++++ ++ ++ ++++++++ * simply is not carried and its stripe frees the slow way. Caller holds
++++++++++++ ++ ++ ++++++++ * the inode's log_mutex, which serializes against the fsync-entry consumer.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static void log_carry_record(struct btrfs_inode *inode, u64 file_start,
++++++++++++ ++ ++ ++++++++ u64 file_len, u64 bytenr, u64 num_bytes,
++++++++++++ ++ ++ ++++++++ u64 ps_start, u64 ps_len)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_log_carry *carry = inode->log_carry;
++++++++++++ ++ ++ ++++++++ int n, i;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!carry) {
++++++++++++ ++ ++ ++++++++ carry = kzalloc(sizeof(*carry), GFP_NOFS);
++++++++++++ ++ ++ ++++++++ if (!carry)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ inode->log_carry = carry;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (ps_len) {
++++++++++++ ++ ++ ++++++++ n = min_t(int, carry->nr_ps, BTRFS_LOG_CARRY_NR_PS);
++++++++++++ ++ ++ ++++++++ for (i = 0; i < n; i++)
++++++++++++ ++ ++ ++++++++ if (carry->ps_start[i] == ps_start)
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ if (i == n) {
++++++++++++ ++ ++ ++++++++ int slot = carry->nr_ps % BTRFS_LOG_CARRY_NR_PS;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ carry->ps_start[slot] = ps_start;
++++++++++++ ++ ++ ++++++++ carry->ps_len[slot] = ps_len;
++++++++++++ ++ ++ ++++++++ carry->nr_ps++;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (!file_len || carry->nr_ranges >= BTRFS_LOG_CARRY_NR_RANGES)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ n = min_t(int, carry->nr_ps, BTRFS_LOG_CARRY_NR_PS);
++++++++++++ ++ ++ ++++++++ for (i = 0; i < n; i++)
++++++++++++ ++ ++ ++++++++ if (bytenr < carry->ps_start[i] + carry->ps_len[i] &&
++++++++++++ ++ ++ ++++++++ bytenr + num_bytes > carry->ps_start[i])
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ if (i == n)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ for (i = 0; i < carry->nr_ranges; i++)
++++++++++++ ++ ++ ++++++++ if (carry->range[i].start == file_start &&
++++++++++++ ++ ++ ++++++++ carry->range[i].len == file_len)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ carry->range[carry->nr_ranges].start = file_start;
++++++++++++ ++ ++ ++++++++ carry->range[carry->nr_ranges].len = file_len;
++++++++++++ ++ ++ ++++++++ carry->range[carry->nr_ranges].disk_bytenr = bytenr;
++++++++++++ ++ ++ ++++++++ carry->nr_ranges++;
++++++++++++ ++ ++ ++++++++ atomic64_inc(&inode->root->fs_info->stripe_run_stats.log_carry_ranges);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Disk end of @inode's last regular file extent, or 0: the re-adoption
++++++++++++ ++ ++ ++++++++ * candidate for a nocow allocation arriving with no allocation hint --
++++++++++++ ++ ++ ++++++++ * after a remount or eviction the extent maps are gone, so the normal
++++++++++++ ++ ++ ++++++++ * em-based hint is empty exactly when re-adoption matters most.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++u64 btrfs_nocow_alloc_hint(struct btrfs_inode *inode)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_root *root = inode->root;
++++++++++++ ++ ++ ++++++++ struct btrfs_path *path;
++++++++++++ ++ ++ ++++++++ struct btrfs_key key;
++++++++++++ ++ ++ ++++++++ struct btrfs_file_extent_item *fi;
++++++++++++ ++ ++ ++++++++ u64 hint = 0;
++++++++++++ ++ ++ ++++++++ int ret;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ path = btrfs_alloc_path();
++++++++++++ ++ ++ ++++++++ if (!path)
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++ key.objectid = btrfs_ino(inode);
++++++++++++ ++ ++ ++++++++ key.type = BTRFS_EXTENT_DATA_KEY;
++++++++++++ ++ ++ ++++++++ key.offset = (u64)-1;
++++++++++++ ++ ++ ++++++++ ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
++++++++++++ ++ ++ ++++++++ if (ret < 0)
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ ret = btrfs_previous_item(root, path, key.objectid,
++++++++++++ ++ ++ ++++++++ BTRFS_EXTENT_DATA_KEY);
++++++++++++ ++ ++ ++++++++ if (ret)
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
++++++++++++ ++ ++ ++++++++ fi = btrfs_item_ptr(path->nodes[0], path->slots[0],
++++++++++++ ++ ++ ++++++++ struct btrfs_file_extent_item);
++++++++++++ ++ ++ ++++++++ if (btrfs_file_extent_type(path->nodes[0], fi) ==
++++++++++++ ++ ++ ++++++++ BTRFS_FILE_EXTENT_INLINE)
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ if (btrfs_file_extent_disk_bytenr(path->nodes[0], fi))
++++++++++++ ++ ++ ++++++++ hint = btrfs_file_extent_disk_bytenr(path->nodes[0], fi) +
++++++++++++ ++ ++ ++++++++ btrfs_file_extent_disk_num_bytes(path->nodes[0], fi);
++++++++++++ ++ ++ ++++++++out:
++++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
++++++++++++ ++ ++ ++++++++ return hint;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/* Does no run object (open or draining) overlap [start, start + len)? */
++++++++++++ ++ ++ ++++++++static bool stripe_range_run_free(struct btrfs_block_group *bg, u64 start,
++++++++++++ ++ ++ ++++++++ u64 len)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ bool ret = true;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (run->start < start + len && run->end > start) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/* Does any stripe run, open or draining, still cover part of @bg? */
++++++++++++ ++ ++ ++++++++bool btrfs_block_group_has_stripe_runs(struct btrfs_block_group *bg)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ bool ret;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ ret = !list_empty(&bg->open_stripe_runs);
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Close @inode's open LOG and NOCOW runs at eviction, so a persistent NOCOW
++++++++++++ ++ ++ ++++++++ * run's claimed tail does not stay unusable for the rest of the mount. The
++++++++++++ ++ ++ ++++++++ * tails return to the free space cache.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_close_inode_stripe_runs(struct btrfs_inode *inode)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = inode->root->fs_info;
++++++++++++ ++ ++ ++++++++ const u64 root_id = btrfs_root_id(inode->root);
++++++++++++ ++ ++ ++++++++ const u64 ino = btrfs_ino(inode);
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++restart:
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs,
++++++++++++ ++ ++ ++++++++ open_stripe_bg_list) {
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ u64 tail_start = 0;
++++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (!run->open || run->root != root_id ||
++++++++++++ ++ ++ ++++++++ run->owner != ino ||
++++++++++++ ++ ++ ++++++++ (run->class != BTRFS_STRIPE_RUN_LOG &&
++++++++++++ ++ ++ ++++++++ run->class != BTRFS_STRIPE_RUN_NOCOW))
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (tail_len) {
++++++++++++ ++ ++ ++++++++ btrfs_get_block_group(bg);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ goto restart;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * A log commit on a shared LOG run: instead of closing the run, move its
++++++++++++ ++ ++ ++++++++ * frontier past the stripe holding the end of the logged extent, so nothing
++++++++++++ ++ ++ ++++++++ * allocates beside the extent after the log commit while the run goes on in
++++++++++++ ++ ++ ++++++++ * the next stripe. The skipped sectors are not live (the pad oracle fills
++++++++++++ ++ ++ ++++++++ * them) and go back to the free space cache like a closed run's tail: inside
++++++++++++ ++ ++ ++++++++ * a partly live stripe they count as trapped until the stripe frees, and a
++++++++++++ ++ ++ ++++++++ * later close returns only [offset, end), which no longer includes them.
++++++++++++ ++ ++ ++++++++ * Caller holds bg->stripe_run_lock. Returns the skipped length.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static u64 seal_log_stripe(struct btrfs_block_group *bg,
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run, u64 ext_end,
++++++++++++ ++ ++ ++++++++ u64 *skip_start)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
++++++++++++ ++ ++ ++++++++ u64 boundary;
++++++++++++ ++ ++ ++++++++ u64 skip;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!run->open)
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++ boundary = bg->start + stripe_round_down(ext_end - 1 - bg->start, fsl) + fsl;
++++++++++++ ++ ++ ++++++++ if (run->offset >= boundary)
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++ boundary = min(boundary, run->end);
++++++++++++ ++ ++ ++++++++ skip = boundary - run->offset;
++++++++++++ ++ ++ ++++++++ *skip_start = run->offset;
++++++++++++ ++ ++ ++++++++ run->offset = boundary;
++++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= skip;
++++++++++++ ++ ++ ++++++++ if (run->offset == run->end) {
++++++++++++ ++ ++ ++++++++ int class;
++++++++++++ ++ ++ ++++++++ int band;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
++++++++++++ ++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
++++++++++++ ++ ++ ++++++++ if (bg->open_stripe[class][band] == run)
++++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
++++++++++++ ++ ++ ++++++++ run->open = false;
++++++++++++ ++ ++ ++++++++ if (open_stripe_run_drained(run) && !run->retained)
++++++++++++ ++ ++ ++++++++ free_open_stripe_run(bg, run);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ return skip;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++static bool run_unwritten_in(struct btrfs_open_stripe_run *run, u64 start,
++++++++++++ ++ ++ ++++++++ u64 len);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Per-stripe settle (stripe_log_data_shared): every allocated sector of the
++++++++++++ ++ ++ ++++++++ * stripes holding [bytenr, bytenr + num_bytes) has been written, or no run
++++++++++++ ++ ++ ++++++++ * covers them any more. Writers sharing those stripes park and merge into
++++++++++++ ++ ++ ++++++++ * the same full-stripe write, so this waits for them and for nothing else in
++++++++++++ ++ ++ ++++++++ * the run.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static bool stripe_log_extent_written(struct btrfs_block_group *bg,
++++++++++++ ++ ++ ++++++++ u64 bytenr, u64 num_bytes)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
++++++++++++ ++ ++ ++++++++ const u64 s = bg->start + stripe_round_down(bytenr - bg->start, fsl);
++++++++++++ ++ ++ ++++++++ const u64 e = bg->start +
++++++++++++ ++ ++ ++++++++ stripe_round_down(bytenr + num_bytes - bg->start + fsl - 1, fsl);
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ bool written = true;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (bytenr < run->start || bytenr >= run->offset)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (run_unwritten_in(run, s, e - s))
++++++++++++ ++ ++ ++++++++ written = false;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ return written;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * A log commit is about to make the extent at [bytenr, bytenr + num_bytes)
++++++++++++ ++ ++ ++++++++ * durable, after which no later write may extend or RMW its stripes. Close
++++++++++++ ++ ++ ++++++++ * the run covering the extent so nothing more allocates into its stripes,
++++++++++++ ++ ++ ++++++++ * kick any parked partial writes there, and wait for all IO into the run to
++++++++++++ ++ ++ ++++++++ * finish. This is a pure data-IO wait (no transaction joins), bounded by
++++++++++++ ++ ++ ++++++++ * bio flight time because parked writes are flushed first, and is safe
++++++++++++ ++ ++ ++++++++ * under the inode log mutex and a running transaction handle.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * With stripe_log_data_shared a shared LOG run is not closed: the stripe
++++++++++++ ++ ++ ++++++++ * holding the end of the extent is sealed (seal_log_stripe()). In that
++++++++++++ ++ ++ ++++++++ * mode the wait, for any run, is for every allocated sector of the
++++++++++++ ++ ++ ++++++++ * extent's stripes to be written (stripe_log_extent_written()) rather than
++++++++++++ ++ ++ ++++++++ * for the run to drain. Neither wait has a timeout: returning early would
++++++++++++ ++ ++ ++++++++ * let a neighbour's partial write RMW the extent after it is durable.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * [file_start, file_start + file_len) is the extent's file range, recorded
++++++++++++ ++ ++ ++++++++ * for carry-forward when the extent is in the run's final partial stripe
++++++++++++ ++ ++ ++++++++ * (stripe_log_data_shared clear only); file_len == 0 opts out of carrying.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_log_settle_stripes(struct btrfs_inode *inode, u64 file_start,
++++++++++++ ++ ++ ++++++++ u64 file_len, u64 bytenr, u64 num_bytes)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = inode->root->fs_info;
++++++++++++ ++ ++ ++++++++ /* Once: the seal and the wait below must agree on the mode. */
++++++++++++ ++ ++ ++++++++ const bool shared = READ_ONCE(fs_info->stripe_log_data_shared);
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ u64 tail_start = 0;
++++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
++++++++++++ ++ ++ ++++++++ u64 flush_start = 0;
++++++++++++ ++ ++ ++++++++ u64 flush_len = 0;
++++++++++++ ++ ++ ++++++++ u64 ps_start = 0;
++++++++++++ ++ ++ ++++++++ u64 ps_len = 0;
++++++++++++ ++ ++ ++++++++ u64 seal_start = 0;
++++++++++++ ++ ++ ++++++++ u64 seal_len = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* stripe_alloc_allow_rmw=fsync waives the log-commit guarantee. */
++++++++++++ ++ ++ ++++++++ if (btrfs_stripe_allow_rmw(fs_info) & BTRFS_STRIPE_RMW_FSYNC)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
++++++++++++ ++ ++ ++++++++ if (!bg)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ u64 filled;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (bytenr < run->start || bytenr >= run->offset)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * A nodatacow file's logged extents get no fsync survival
++++++++++++ ++ ++ ++++++++ * guarantee -- the file's own later in-place writes can
++++++++++++ ++ ++ ++++++++ * always tear them -- so closing its run would trap the
++++++++++++ ++ ++ ++++++++ * tail for nothing.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (run->class == BTRFS_STRIPE_RUN_NOCOW)
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ if (shared && (run->owner & BIT_ULL(63))) {
++++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* only the logged extent's stripes, not the shared run */
++++++++++++ ++ ++ ++++++++ flush_start = bg->start + stripe_round_down(bytenr - bg->start, fsl);
++++++++++++ ++ ++ ++++++++ flush_len = bg->start + stripe_round_down(bytenr + num_bytes - 1 - bg->start, fsl) +
++++++++++++ ++ ++ ++++++++ fsl - flush_start;
++++++++++++ ++ ++ ++++++++ seal_len = seal_log_stripe(bg, run, bytenr + num_bytes, &seal_start);
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ flush_start = run->start;
++++++++++++ ++ ++ ++++++++ flush_len = run->end - run->start;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * The run's final partial stripe, whether this call closes
++++++++++++ ++ ++ ++++++++ * the run or an earlier settle already did (offset froze at
++++++++++++ ++ ++ ++++++++ * close). Runs start stripe-aligned, so alignment relative
++++++++++++ ++ ++ ++++++++ * to run->start is stripe alignment.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ filled = run->offset - run->start;
++++++++++++ ++ ++ ++++++++ if (!stripe_is_aligned(filled, bg->full_stripe_len)) {
++++++++++++ ++ ++ ++++++++ ps_start = run->start +
++++++++++++ ++ ++ ++++++++ stripe_round_down(filled, bg->full_stripe_len);
++++++++++++ ++ ++ ++++++++ ps_len = bg->full_stripe_len;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (run->open) {
++++++++++++ ++ ++ ++++++++ int class;
++++++++++++ ++ ++ ++++++++ int band;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
++++++++++++ ++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
++++++++++++ ++ ++ ++++++++ if (bg->open_stripe[class][band] == run)
++++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
++++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!shared)
++++++++++++ ++ ++ ++++++++ log_carry_record(inode, file_start, file_len, bytenr, num_bytes,
++++++++++++ ++ ++ ++++++++ ps_start, ps_len);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (seal_len) {
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, seal_start, seal_len);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.log_seals);
++++++++++++ ++ ++ ++++++++ atomic64_add(seal_len, &fs_info->stripe_run_stats.log_seal_bytes);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (tail_len) {
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (flush_len) {
++++++++++++ ++ ++ ++++++++ ktime_t t0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, flush_start, flush_len, false);
++++++++++++ ++ ++ ++++++++ t0 = ktime_get();
++++++++++++ ++ ++ ++++++++ if (shared)
++++++++++++ ++ ++ ++++++++ wait_var_event(&bg->open_stripe_runs,
++++++++++++ ++ ++ ++++++++ stripe_log_extent_written(bg, bytenr, num_bytes) ||
++++++++++++ ++ ++ ++++++++ BTRFS_FS_ERROR(fs_info));
++++++++++++ ++ ++ ++++++++ else
++++++++++++ ++ ++ ++++++++ wait_var_event(&bg->open_stripe_runs,
++++++++++++ ++ ++ ++++++++ stripe_log_range_settled(bg, bytenr) ||
++++++++++++ ++ ++ ++++++++ BTRFS_FS_ERROR(fs_info));
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.log_settle_waits);
++++++++++++ ++ ++ ++++++++ atomic64_add(ktime_to_ns(ktime_sub(ktime_get(), t0)),
++++++++++++ ++ ++ ++++++++ &fs_info->stripe_run_stats.log_settle_wait_ns);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Can stripe-exclusive allocation be enabled on this filesystem? Called for
++++++++++++ ++ ++ ++++++++ * the "stripe_alloc" filesystem property, on a mounted (or mounting, options
++++++++++++ ++ ++ ++++++++ * finalized) filesystem.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++int btrfs_stripe_alloc_check_support(struct btrfs_fs_info *fs_info)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Only the v1 space cache is a problem. Its cache inode is nodatacow
++++++++++++ ++ ++ ++++++++ * and preallocated, so a commit overwrites it in place -- a sub-stripe
++++++++++++ ++ ++ ++++++++ * write into committed stripes that no csum would show afterwards.
++++++++++++ ++ ++ ++++++++ * The free space tree and no cache at all are both fine; nothing here
++++++++++++ ++ ++ ++++++++ * reads either one.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Cache inodes left on disk by an earlier v1 mount do not matter:
++++++++++++ ++ ++ ++++++++ * only cache_save_setup() writes them, and only with SPACE_CACHE.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (btrfs_test_opt(fs_info, SPACE_CACHE)) {
++++++++++++ ++ ++ ++++++++ btrfs_err(fs_info,
++++++++++++ ++ ++ ++++++++ "stripe_alloc is not supported with space_cache=v1");
++++++++++++ ++ ++ ++++++++ return -EINVAL;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (btrfs_fs_incompat(fs_info, REMAP_TREE)) {
++++++++++++ ++ ++ ++++++++ btrfs_err(fs_info,
++++++++++++ ++ ++ ++++++++ "stripe_alloc is not supported together with the remap-tree feature");
++++++++++++ ++ ++ ++++++++ return -EINVAL;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (btrfs_is_zoned(fs_info)) {
++++++++++++ ++ ++ ++++++++ btrfs_err(fs_info,
++++++++++++ ++ ++ ++++++++ "stripe_alloc is not supported on zoned filesystems");
++++++++++++ ++ ++ ++++++++ return -EINVAL;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * A mixed group is excluded from the claim rule anyway
++++++++++++ ++ ++ ++++++++ * (btrfs_is_stripe_alloc_bg() requires DATA without METADATA), so
++++++++++++ ++ ++ ++++++++ * enabling here would protect nothing while reporting success.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (btrfs_fs_incompat(fs_info, MIXED_GROUPS)) {
++++++++++++ ++ ++ ++++++++ btrfs_err(fs_info,
++++++++++++ ++ ++ ++++++++ "stripe_alloc is not supported on mixed block groups");
++++++++++++ ++ ++ ++++++++ return -EINVAL;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Arm or disarm stripe_unusable accounting across all raid56 data and
++++++++++++ ++ ++ ++++++++ * metadata block groups whose caches are loaded. Same walk as the
++++++++++++ ++ ++ ++++++++ * commit-time rescan.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static void stripe_alloc_sweep_groups(struct btrfs_fs_info *fs_info, bool arm)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_space_info *sinfo;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ list_for_each_entry(sinfo, &fs_info->space_info, list) {
++++++++++++ ++ ++ ++++++++ int raid;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!(sinfo->flags & (BTRFS_BLOCK_GROUP_DATA |
++++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_METADATA |
++++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Metadata: the scan that maintains the stripe_meta margin
++++++++++++ ++ ++ ++++++++ * and reserve stops with the option, so clear them here or
++++++++++++ ++ ++ ++++++++ * they hold metadata back forever. The groups themselves
++++++++++++ ++ ++ ++++++++ * are armed and disarmed below like data groups: raid56
++++++++++++ ++ ++ ++++++++ * metadata places tree blocks in whole stripes from the
++++++++++++ ++ ++ ++++++++ * moment the option is on, so its trapped and claimable
++++++++++++ ++ ++ ++++++++ * bytes count from that moment and stop counting when it
++++++++++++ ++ ++ ++++++++ * goes off.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (!arm && (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA)) {
++++++++++++ ++ ++ ++++++++ spin_lock(&sinfo->lock);
++++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_margin = 0;
++++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_reserve = 0;
++++++++++++ ++ ++ ++++++++ btrfs_try_granting_tickets(sinfo);
++++++++++++ ++ ++ ++++++++ spin_unlock(&sinfo->lock);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ /* The writeback quarantine is the gate's; it goes with it. */
++++++++++++ ++ ++ ++++++++ if (!arm && (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)) {
++++++++++++ ++ ++ ++++++++ spin_lock(&sinfo->lock);
++++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_quarantine = 0;
++++++++++++ ++ ++ ++++++++ btrfs_try_granting_tickets(sinfo);
++++++++++++ ++ ++ ++++++++ spin_unlock(&sinfo->lock);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ down_read(&sinfo->groups_sem);
++++++++++++ ++ ++ ++++++++ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!(btrfs_raid_array[raid].bg_flag &
++++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_RAID56_MASK))
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &sinfo->block_groups[raid],
++++++++++++ ++ ++ ++++++++ list) {
++++++++++++ ++ ++ ++++++++ if (!arm)
++++++++++++ ++ ++ ++++++++ btrfs_block_group_disarm_stripe_unusable(bg);
++++++++++++ ++ ++ ++++++++ else if (bg->cached == BTRFS_CACHE_FINISHED)
++++++++++++ ++ ++ ++++++++ btrfs_block_group_init_stripe_unusable(bg);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ up_read(&sinfo->groups_sem);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Drain the writes the legacy allocator placed before stripe_alloc was
++++++++++++ ++ ++ ++++++++ * enabled on a live filesystem, then arm the write-hole invariant check.
++++++++++++ ++ ++ ++++++++ * Runs in a worker: the property that enables the policy is applied inside
++++++++++++ ++ ++ ++++++++ * a transaction, where flushing delalloc cannot wait.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_stripe_alloc_enable_work(struct work_struct *work)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = container_of(work, struct btrfs_fs_info,
++++++++++++ ++ ++ ++++++++ stripe_alloc_enable_work);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ btrfs_start_delalloc_roots(fs_info, LONG_MAX, false);
++++++++++++ ++ ++ ++++++++ btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Tree blocks the legacy allocator placed in the transaction running at
++++++++++++ ++ ++ ++++++++ * the flip are written into partly used stripes by that transaction's
++++++++++++ ++ ++ ++++++++ * commit. Wait for it before arming the check, which would otherwise
++++++++++++ ++ ++ ++++++++ * report those writes as violations.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ btrfs_commit_current_transaction(fs_info->tree_root);
++++++++++++ ++ ++ ++++++++ clear_bit(BTRFS_FS_STRIPE_ALLOC_ENABLING, &fs_info->flags);
++++++++++++ ++ ++ ++++++++ btrfs_info(fs_info,
++++++++++++ ++ ++ ++++++++ "stripe_alloc: writes placed before the policy was enabled have landed");
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Turn on stripe-exclusive allocation, from the "stripe_alloc" filesystem
++++++++++++ ++ ++ ++++++++ * property on the top-level subvolume's root directory. Runs during mount
++++++++++++ ++ ++ ++++++++ * when the root directory inode loads its properties -- before any user IO
++++++++++++ ++ ++ ++++++++ * -- or at runtime from btrfs_set_prop().
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++int btrfs_enable_stripe_alloc(struct btrfs_fs_info *fs_info)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ bool live = test_bit(BTRFS_FS_OPEN, &fs_info->flags);
++++++++++++ ++ ++ ++++++++ int ret;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (btrfs_test_opt(fs_info, STRIPE_ALLOC))
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++ ret = btrfs_stripe_alloc_check_support(fs_info);
++++++++++++ ++ ++ ++++++++ if (ret)
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * On a live filesystem the legacy allocator may still have writes in
++++++++++++ ++ ++ ++++++++ * flight. Their extents sit in partly used stripes that no stripe run
++++++++++++ ++ ++ ++++++++ * covers, so the write-hole invariant check would report them although
++++++++++++ ++ ++ ++++++++ * they were placed before the policy existed. Keep the check off until
++++++++++++ ++ ++ ++++++++ * every ordered extent that could have been allocated before the flip
++++++++++++ ++ ++ ++++++++ * has landed; extents allocated after it have stripe runs. A mount-time
++++++++++++ ++ ++ ++++++++ * enable has nothing in flight and skips the drain.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (live)
++++++++++++ ++ ++ ++++++++ set_bit(BTRFS_FS_STRIPE_ALLOC_ENABLING, &fs_info->flags);
++++++++++++ ++ ++ ++++++++ btrfs_set_opt(fs_info->mount_opt, STRIPE_ALLOC);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Arm stripe_unusable accounting for raid56 data groups whose caches
++++++++++++ ++ ++ ++++++++ * loaded before the policy was enabled: block groups are read before
++++++++++++ ++ ++ ++++++++ * the root directory's properties apply during mount, and a runtime
++++++++++++ ++ ++ ++++++++ * property set can arrive at any point. Groups still caching arm
++++++++++++ ++ ++ ++++++++ * when their caching thread finishes, now that the policy is on.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ stripe_alloc_sweep_groups(fs_info, true);
++++++++++++ ++ ++ ++++++++ /* At mount, btrfs_fill_super() reports the outcome. */
++++++++++++ ++ ++ ++++++++ if (live) {
++++++++++++ ++ ++ ++++++++ btrfs_info(fs_info,
++++++++++++ ++ ++ ++++++++ "using stripe-exclusive allocation for raid56 data (filesystem property)");
++++++++++++ ++ ++ ++++++++ queue_work(system_unbound_wq, &fs_info->stripe_alloc_enable_work);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ return 0;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Turn off stripe-exclusive allocation (property deleted). Open runs are
++++++++++++ ++ ++ ++++++++ * not touched here: the commit-time retirement runs unconditionally and
++++++++++++ ++ ++ ++++++++ * drains them at the next transaction commit.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_disable_stripe_alloc(struct btrfs_fs_info *fs_info)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ /* Disarm while the policy bit still passes the per-group gate. */
++++++++++++ ++ ++ ++++++++ stripe_alloc_sweep_groups(fs_info, false);
++++++++++++ ++ ++ ++++++++ btrfs_clear_opt(fs_info->mount_opt, STRIPE_ALLOC);
++++++++++++ ++ ++ ++++++++ btrfs_info(fs_info,
++++++++++++ ++ ++ ++++++++ "stripe-exclusive allocation for raid56 data disabled");
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Does @logical lie within a stripe run, open or draining? The raid56 layer
++++++++++++ ++ ++ ++++++++ * uses this to decide whether a partial write may be parked to merge into a
++++++++++++ ++ ++ ++++++++ * full-stripe write. An open run fills the rest of the stripe later or
++++++++++++ ++ ++ ++++++++ * never; a draining run's frontier is frozen, so everything below it is IO
++++++++++++ ++ ++ ++++++++ * the commit drain already waits for and everything above it is dead space
++++++++++++ ++ ++ ++++++++ * padding may fill.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++bool btrfs_stripe_in_open_run(struct btrfs_fs_info *fs_info, u64 logical)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ return btrfs_stripe_open_run_class(fs_info, logical, NULL);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * The part of [*start, *start + *len) that @run owns and tracks in its
++++++++++++ ++ ++ ++++++++ * written map: inside [run->start, run->end) -- the map itself reaches
++++++++++++ ++ ++ ++++++++ * further (it spans at least BTRFS_STRIPE_RUN_LIVE_STRIPES stripes from
++++++++++++ ++ ++ ++++++++ * run->start, past run->end into later runs' ranges), so it must never be
++++++++++++ ++ ++ ++++++++ * consulted outside the run's own range -- and inside the map. Nodatacow
++++++++++++ ++ ++ ++++++++ * runs keep no written map: they are rewritten in place for as long as the
++++++++++++ ++ ++ ++++++++ * inode lives and are never complete. Caller holds stripe_run_lock.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static bool run_written_range(const struct btrfs_open_stripe_run *run,
++++++++++++ ++ ++ ++++++++ u64 *start, u64 *len)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
++++++++++++ ++ ++ ++++++++ u64 lo, hi;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (run->class == BTRFS_STRIPE_RUN_NOCOW)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ if (!run->live || !run->written || !run->live_nbits)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ lo = max(*start, run->start);
++++++++++++ ++ ++ ++++++++ hi = min3(*start + *len, run->end,
++++++++++++ ++ ++ ++++++++ run->start + ((u64)run->live_nbits << bits));
++++++++++++ ++ ++ ++++++++ if (lo >= hi)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ *start = lo;
++++++++++++ ++ ++ ++++++++ *len = hi - lo;
++++++++++++ ++ ++ ++++++++ return true;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * A raid56 write of [start, start + len) has completed: mark it written in
++++++++++++ ++ ++ ++++++++ * the run that owns it. Sectors that are not live in that run (freed while
++++++++++++ ++ ++ ++++++++ * the write was in flight, or a bookkeeping bug) are counted in
++++++++++++ ++ ++ ++++++++ * written_not_live: the pad oracle trusts this map to say which allocated
++++++++++++ ++ ++ ++++++++ * sectors hold data, so a write it misses could be zeroed.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_stripe_run_mark_written(struct btrfs_fs_info *fs_info, u64 start,
++++++++++++ ++ ++ ++++++++ u64 len)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ const u32 bits = fs_info->sectorsize_bits;
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ u32 not_live = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, start);
++++++++++++ ++ ++ ++++++++ if (!bg)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ u64 s = start, l = len;
++++++++++++ ++ ++ ++++++++ u32 first, nr, i;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!run_written_range(run, &s, &l))
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ first = (s - run->start) >> bits;
++++++++++++ ++ ++ ++++++++ nr = l >> bits;
++++++++++++ ++ ++ ++++++++ for (i = first; i < first + nr; i++)
++++++++++++ ++ ++ ++++++++ if (!test_bit(i, run->live))
++++++++++++ ++ ++ ++++++++ not_live++;
++++++++++++ ++ ++ ++++++++ bitmap_set(run->written, first, nr);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ /* Per-stripe log settles wait on the written map, not on a drain. */
++++++++++++ ++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ if (not_live)
++++++++++++ ++ ++ ++++++++ atomic64_add(not_live, &fs_info->stripe_park_stats.written_not_live);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/* Has @run allocated sectors in [start, start + len) it has not yet written? */
++++++++++++ ++ ++ ++++++++static bool run_unwritten_in(struct btrfs_open_stripe_run *run, u64 start,
++++++++++++ ++ ++ ++++++++ u64 len)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
++++++++++++ ++ ++ ++++++++ u32 first, last, i;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!run_written_range(run, &start, &len))
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ first = (start - run->start) >> bits;
++++++++++++ ++ ++ ++++++++ last = first + (len >> bits);
++++++++++++ ++ ++ ++++++++ for (i = first; i < last; i++)
++++++++++++ ++ ++ ++++++++ if (test_bit(i, run->live) && !test_bit(i, run->written))
++++++++++++ ++ ++ ++++++++ return true;
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * As btrfs_stripe_run_incomplete(), returning the class of the run that
++++++++++++ ++ ++ ++++++++ * leaves the stripe incomplete, or -1 if none does.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++int btrfs_stripe_run_incomplete_class(struct btrfs_fs_info *fs_info,
++++++++++++ ++ ++ ++++++++ u64 stripe_start, u64 stripe_len)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ int ret = -1;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
++++++++++++ ++ ++ ++++++++ return -1;
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
++++++++++++ ++ ++ ++++++++ if (!bg)
++++++++++++ ++ ++ ++++++++ return -1;
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if ((run->open && run->offset > stripe_start &&
++++++++++++ ++ ++ ++++++++ run->offset < stripe_start + stripe_len) ||
++++++++++++ ++ ++ ++++++++ run_unwritten_in(run, stripe_start, stripe_len)) {
++++++++++++ ++ ++ ++++++++ ret = run->class;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Is [stripe_start, stripe_start + stripe_len) still waiting for writes from
++++++++++++ ++ ++ ++++++++ * a stripe run? Two ways:
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * - an open run's allocation frontier lies strictly inside it: the run will
++++++++++++ ++ ++ ++++++++ * allocate and write the rest of the stripe later, although every sector
++++++++++++ ++ ++ ++++++++ * allocated so far may already be written;
++++++++++++ ++ ++ ++++++++ * - a run covering it has sectors allocated (live) but not yet written: the
++++++++++++ ++ ++ ++++++++ * frontier has moved past, and the writes are still in flight (writeback
++++++++++++ ++ ++ ++++++++ * allocates a whole range before any of its bios complete).
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Either way the stripe has been written in part, and the rest will arrive
++++++++++++ ++ ++ ++++++++ * as a sub-stripe write that needs the part already written.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++bool btrfs_stripe_run_incomplete(struct btrfs_fs_info *fs_info,
++++++++++++ ++ ++ ++++++++ u64 stripe_start, u64 stripe_len)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ return btrfs_stripe_run_incomplete_class(fs_info, stripe_start,
++++++++++++ ++ ++ ++++++++ stripe_len) >= 0;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/* Open stripe runs by class, for sysfs: @counts has NR_CLASSES entries. */
++++++++++++ ++ ++ ++++++++void btrfs_count_open_stripe_runs(struct btrfs_fs_info *fs_info, int *counts)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ memset(counts, 0, sizeof(*counts) * BTRFS_STRIPE_RUN_NR_CLASSES);
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list) {
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list)
++++++++++++ ++ ++ ++++++++ if (run->open && run->class < BTRFS_STRIPE_RUN_NR_CLASSES)
++++++++++++ ++ ++ ++++++++ counts[run->class]++;
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Like btrfs_stripe_in_open_run(), also reporting the covering run's
++++++++++++ ++ ++ ++++++++ * class: the raid56 parking gate treats NOCOW-class runs specially
++++++++++++ ++ ++ ++++++++ * (sync writes into them never park; see rbio_try_park()).
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++bool btrfs_stripe_open_run_class(struct btrfs_fs_info *fs_info, u64 logical,
++++++++++++ ++ ++ ++++++++ enum btrfs_stripe_run_class *class)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ bool ret = false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, logical);
++++++++++++ ++ ++ ++++++++ if (!bg)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (logical < run->start || logical >= run->end)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Draining runs match too (see btrfs_stripe_in_open_run()):
++++++++++++ ++ ++ ++++++++ * a closed run's end is shrunk to its frontier, so a write
++++++++++++ ++ ++ ++++++++ * into it is below the frontier and its stripes are still
++++++++++++ ++ ++ ++++++++ * exclusive.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (class)
++++++++++++ ++ ++ ++++++++ *class = run->class;
++++++++++++ ++ ++ ++++++++ ret = true;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Report bytes that will never be written: an allocation released before
++++++++++++ ++ ++ ++++++++ * its write was issued. Same accounting as a completion, plus the sectors
++++++++++++ ++ ++ ++++++++ * go dead so a later partial write to their stripe can pad over them
++++++++++++ ++ ++ ++++++++ * instead of falling back to read-modify-write.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_open_stripe_write_abandoned(struct btrfs_block_group *bg, u64 start,
++++++++++++ ++ ++ ++++++++ u64 num_bytes)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (start >= run->start && start < run->offset) {
++++++++++++ ++ ++ ++++++++ ASSERT(start + num_bytes <= run->offset);
++++++++++++ ++ ++ ++++++++ ASSERT(run->inflight_bytes >= num_bytes);
++++++++++++ ++ ++ ++++++++ run_live_set(run, start, num_bytes, false);
++++++++++++ ++ ++ ++++++++ run->inflight_bytes -= num_bytes;
++++++++++++ ++ ++ ++++++++ open_stripe_run_drain_reported(bg, run);
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Report completed IO for an allocation from any stripe run. Every
++++++++++++ ++ ++ ++++++++ * allocated byte must be reported exactly once, here or through
++++++++++++ ++ ++ ++++++++ * btrfs_open_stripe_write_abandoned(), in any number of pieces (e.g. after
++++++++++++ ++ ++ ++++++++ * ordered extent splits).
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_open_stripe_write_done(struct btrfs_block_group *bg, u64 start,
++++++++++++ ++ ++ ++++++++ u64 num_bytes)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (start >= run->start && start < run->offset) {
++++++++++++ ++ ++ ++++++++ ASSERT(start + num_bytes <= run->offset);
++++++++++++ ++ ++ ++++++++ ASSERT(run->inflight_bytes >= num_bytes);
++++++++++++ ++ ++ ++++++++ run->inflight_bytes -= num_bytes;
++++++++++++ ++ ++ ++++++++ open_stripe_run_drain_reported(bg, run);
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Range variant of btrfs_open_stripe_write_done() for callers without the
++++++++++++ ++ ++ ++++++++ * block group at hand (e.g. the prealloc path, which never issues data IO
++++++++++++ ++ ++ ++++++++ * for its extent and reports it done at insertion).
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_open_stripe_write_done_bytenr(struct btrfs_fs_info *fs_info,
++++++++++++ ++ ++ ++++++++ u64 start, u64 num_bytes)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, start);
++++++++++++ ++ ++ ++++++++ if (!bg)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ btrfs_open_stripe_write_done(bg, start, num_bytes);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Find the open stripe run whose allocated region contains
++++++++++++ ++ ++ ++++++++ * [bytenr, bytenr + num_bytes). The caller must own unreported inflight
++++++++++++ ++ ++ ++++++++ * bytes in that region (i.e. it was allocated from the run and not yet
++++++++++++ ++ ++ ++++++++ * reported done), which is what keeps the returned pointer valid: a run
++++++++++++ ++ ++ ++++++++ * with inflight bytes is never freed.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++struct btrfs_open_stripe_run *btrfs_get_open_stripe_run(
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info, u64 bytenr, u64 num_bytes,
++++++++++++ ++ ++ ++++++++ u64 *open_seq)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run = NULL;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *iter;
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
++++++++++++ ++ ++ ++++++++ if (!bg)
++++++++++++ ++ ++ ++++++++ return NULL;
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(iter, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (bytenr >= iter->start && bytenr < iter->offset) {
++++++++++++ ++ ++ ++++++++ ASSERT(bytenr + num_bytes <= iter->offset);
++++++++++++ ++ ++ ++++++++ ASSERT(iter->inflight_bytes >= num_bytes);
++++++++++++ ++ ++ ++++++++ run = iter;
++++++++++++ ++ ++ ++++++++ *open_seq = iter->open_seq;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ return run;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Pointer-based variant of btrfs_open_stripe_write_done() for callers that
++++++++++++ ++ ++ ++++++++ * recorded the run at allocation or ordered extent creation time. The run
++++++++++++ ++ ++ ++++++++ * pointer stays valid until the reporter's own bytes are reported: a run
++++++++++++ ++ ++ ++++++++ * with inflight bytes is never freed.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * An ordered extent's data IO is over, whether or not all of it reached the
++++++++++++ ++ ++ ++++++++ * disk: stop counting it as in flight, and mark its sectors written. A
++++++++++++ ++ ++ ++++++++ * sector whose write failed or was never issued (truncation, a short direct
++++++++++++ ++ ++ ++++++++ * write, error teardown) will not be written now. Marking it keeps a log
++++++++++++ ++ ++ ++++++++ * settle from waiting for it forever, and a written sector is never padded,
++++++++++++ ++ ++ ++++++++ * so a later partial write to the stripe reads it instead of zeroing it.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Returns the number of allocated sectors that had not been written, and
++++++++++++ ++ ++ ++++++++ * in @first_unwritten the logical address of the lowest of them (U64_MAX if
++++++++++++ ++ ++ ++++++++ * there are none).
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++u32 btrfs_open_stripe_write_done_run(struct btrfs_open_stripe_run *run,
++++++++++++ ++ ++ ++++++++ u64 start, u64 num_bytes,
++++++++++++ ++ ++ ++++++++ u64 *first_unwritten)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg = run->bg;
++++++++++++ ++ ++ ++++++++ const u32 bits = bg->fs_info->sectorsize_bits;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ u64 s = start;
++++++++++++ ++ ++ ++++++++ u64 l = num_bytes;
++++++++++++ ++ ++ ++++++++ u32 unwritten = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ *first_unwritten = U64_MAX;
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ ASSERT(run->inflight_bytes >= num_bytes);
++++++++++++ ++ ++ ++++++++ run->inflight_bytes -= num_bytes;
++++++++++++ ++ ++ ++++++++ if (run_written_range(run, &s, &l)) {
++++++++++++ ++ ++ ++++++++ const u32 first = (s - run->start) >> bits;
++++++++++++ ++ ++ ++++++++ const u32 last = first + (l >> bits);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ for (u32 i = first; i < last; i++) {
++++++++++++ ++ ++ ++++++++ if (!test_bit(i, run->live) || test_bit(i, run->written))
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (!unwritten)
++++++++++++ ++ ++ ++++++++ *first_unwritten = run->start + ((u64)i << bits);
++++++++++++ ++ ++ ++++++++ unwritten++;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ bitmap_set(run->written, first, l >> bits);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ /* Under the lock: the run and the group may be gone after it. */
++++++++++++ ++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
++++++++++++ ++ ++ ++++++++ open_stripe_run_drain_reported(bg, run);
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ return unwritten;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Close the block group's open runs that predate @seq and return their
++++++++++++ ++ ++ ++++++++ * unallocated tails to the free space cache. Does not wait for inflight
++++++++++++ ++ ++ ++++++++ * IO. Metadata runs are left to btrfs_retire_meta_stripes() and persistent
++++++++++++ ++ ++ ++++++++ * NOCOW runs stay open unless dead, except when @seq is U64_MAX (read-only,
++++++++++++ ++ ++ ++++++++ * removal, unmount), which closes everything.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static void close_block_group_stripe_runs(struct btrfs_block_group *bg,
++++++++++++ ++ ++ ++++++++ u64 seq)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ int class, band;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++restart:
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ u64 tail_start;
++++++++++++ ++ ++ ++++++++ u64 tail_len;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!run->open || run->open_seq >= seq)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * A persistent NOCOW run holds only its inode's
++++++++++++ ++ ++ ++++++++ * write-hole-waived data, so commit retirement protects
++++++++++++ ++ ++ ++++++++ * nothing there, and closing it each commit would waste a
++++++++++++ ++ ++ ++++++++ * stripe per commit on a slowly appended file. It closes at
++++++++++++ ++ ++ ++++++++ * forced quiescing (seq == U64_MAX), at the inode's eviction,
++++++++++++ ++ ++ ++++++++ * or once it is dead.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (run->persistent && seq != U64_MAX) {
++++++++++++ ++ ++ ++++++++ /* A dead run protects nothing, only strands stripes. */
++++++++++++ ++ ++ ++++++++ if (!open_stripe_run_dead(run))
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.nocow_run_dead_closed);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Metadata runs are still being filled at this point in the
++++++++++++ ++ ++ ++++++++ * commit and their tree blocks are not written until later;
++++++++++++ ++ ++ ++++++++ * btrfs_retire_meta_stripes() closes them once they are.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class) && seq != U64_MAX)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
++++++++++++ ++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
++++++++++++ ++ ++ ++++++++ if (bg->open_stripe[class][band] == run)
++++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
++++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Closing may have freed the run, and returning its tail to
++++++++++++ ++ ++ ++++++++ * the free space cache needs the lock dropped, so rescan.
++++++++++++ ++ ++ ++++++++ * Each pass closes one run and open runs are bounded (band
++++++++++++ ++ ++ ++++++++ * slots plus the private runs of inodes logged this window).
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (tail_len)
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ goto restart;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Close all of a block group's open stripe runs without waiting for their
++++++++++++ ++ ++ ++++++++ * inflight IO to drain.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_close_bg_open_stripes(struct btrfs_block_group *bg)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ close_block_group_stripe_runs(bg, U64_MAX);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/* No run opened before @seq remains (open or draining). */
++++++++++++ ++ ++ ++++++++static bool bg_open_stripes_settled(struct btrfs_block_group *bg, u64 seq)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ bool ret = true;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ /* Persistent nocow runs are not part of any commit window. */
++++++++++++ ++ ++ ++++++++ if (run->persistent && seq != U64_MAX)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /* Nor are retained relocation runs, once nothing is in flight. */
++++++++++++ ++ ++ ++++++++ if (run->retained && open_stripe_run_drained(run) &&
++++++++++++ ++ ++ ++++++++ seq != U64_MAX)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /* Metadata drains after tree writeback, not here. */
++++++++++++ ++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class) && seq != U64_MAX)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (run->open_seq < seq) {
++++++++++++ ++ ++ ++++++++ ret = false;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Stop retaining @bg's relocation runs: free the ones already drained, and
++++++++++++ ++ ++ ++++++++ * let any still open or in flight be freed when they drain, like any other.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static void release_retained_stripe_runs(struct btrfs_block_group *bg)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run, *tmp;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry_safe(run, tmp, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (!run->retained)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ run->retained = false;
++++++++++++ ++ ++ ++++++++ atomic64_inc(&bg->fs_info->stripe_run_stats.reloc_run_released);
++++++++++++ ++ ++ ++++++++ if (open_stripe_run_drained(run))
++++++++++++ ++ ++ ++++++++ free_open_stripe_run(bg, run);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Relocation is done with its destination stripes: release its retained
++++++++++++ ++ ++ ++++++++ * runs in every block group (the dedication of btrfs_release_data_reloc_bg()
++++++++++++ ++ ++ ++++++++ * can have moved between groups during the relocation).
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static void btrfs_release_stripe_reloc_runs(struct btrfs_fs_info *fs_info)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
++++++++++++ ++ ++ ++++++++ release_retained_stripe_runs(bg);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/* Does @bg have metadata stripe runs? */
++++++++++++ ++ ++ ++++++++static bool bg_has_meta_runs(struct btrfs_block_group *bg)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ bool ret = false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class)) {
++++++++++++ ++ ++ ++++++++ ret = true;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Close and drain all of a block group's stripe runs. The caller must
++++++++++++ ++ ++ ++++++++ * prevent new allocations first (e.g. the block group is read-only),
++++++++++++ ++ ++ ++++++++ * otherwise this can wait forever. Membership in fs_info->open_stripe_bgs
++++++++++++ ++ ++ ++++++++ * is left for the next retire walk to collect.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * The wait also ends when the filesystem is in error: after a transaction
++++++++++++ ++ ++ ++++++++ * abort the ordered extents holding a run's bytes may never complete. The
++++++++++++ ++ ++ ++++++++ * runs are then left for btrfs_free_open_stripe_runs() at teardown.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_retire_block_group_stripes(struct btrfs_block_group *bg)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
++++++++++++ ++ ++ ++++++++ bool warned = false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Nothing may outlive this, relocation's retained runs included. */
++++++++++++ ++ ++ ++++++++ release_retained_stripe_runs(bg);
++++++++++++ ++ ++ ++++++++ close_block_group_stripe_runs(bg, U64_MAX);
++++++++++++ ++ ++ ++++++++ /* Parked partial-stripe rbios hold the very bios the wait drains. */
++++++++++++ ++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, bg->start, bg->length, false);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Metadata runs drain when their tree blocks are written and are
++++++++++++ ++ ++ ++++++++ * freed by btrfs_retire_meta_stripes(), both in a commit. Commit now
++++++++++++ ++ ++ ++++++++ * instead of waiting for the next one; no caller holds a transaction
++++++++++++ ++ ++ ++++++++ * handle.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (bg_has_meta_runs(bg))
++++++++++++ ++ ++ ++++++++ btrfs_commit_current_transaction(fs_info->tree_root);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * No timeout: a run that does not drain has an allocation that was
++++++++++++ ++ ++ ++++++++ * never reported written or abandoned, an accounting bug to fix, not
++++++++++++ ++ ++ ++++++++ * to wait out. Say which runs once, and keep waiting.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ while (!wait_var_event_timeout(&bg->open_stripe_runs,
++++++++++++ ++ ++ ++++++++ bg_open_stripes_settled(bg, U64_MAX) ||
++++++++++++ ++ ++ ++++++++ BTRFS_FS_ERROR(fs_info), 60 * HZ)) {
++++++++++++ ++ ++ ++++++++ if (warned)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ btrfs_warn(fs_info,
++++++++++++ ++ ++ ++++++++"block group %llu read-only transition waiting for stripe runs that do not drain",
++++++++++++ ++ ++ ++++++++ bg->start);
++++++++++++ ++ ++ ++++++++ btrfs_dump_open_stripe_runs(bg);
++++++++++++ ++ ++ ++++++++ warned = true;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (BTRFS_FS_ERROR(fs_info) && !bg_open_stripes_settled(bg, U64_MAX))
++++++++++++ ++ ++ ++++++++ btrfs_warn(fs_info,
++++++++++++ ++ ++ ++++++++"block group %llu read-only with stripe runs still in flight after a filesystem error; not waiting for them",
++++++++++++ ++ ++ ++++++++ bg->start);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * The filesystem has gone into error: wake every task waiting for a block
++++++++++++ ++ ++ ++++++++ * group's stripe runs to drain, so the waits that give up on an error
++++++++++++ ++ ++ ++++++++ * (btrfs_retire_block_group_stripes()) notice it. A run's own completion
++++++++++++ ++ ++ ++++++++ * reports may never arrive now.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_wake_open_stripe_waiters(struct btrfs_fs_info *fs_info)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * The caller set fs_error without a lock, and wake_up_var() checks
++++++++++++ ++ ++ ++++++++ * for waiters without one: order the store before that check, or a
++++++++++++ ++ ++ ++++++++ * waiter that has just tested BTRFS_FS_ERROR is missed. Pairs with
++++++++++++ ++ ++ ++++++++ * the barrier in prepare_to_wait_event().
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ smp_mb();
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
++++++++++++ ++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Teardown: free every remaining stripe run and drop each group's
++++++++++++ ++ ++ ++++++++ * membership reference. Needed when no final commit retired them: a mount
++++++++++++ ++ ++ ++++++++ * that fails after log replay allocated tree blocks, or an unmount after a
++++++++++++ ++ ++ ++++++++ * transaction abort. IO is quiesced by now, so a run still counting
++++++++++++ ++ ++ ++++++++ * inflight bytes is reported and freed anyway.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Ordered extents point at their runs (ordered->stripe_run), so this relies
++++++++++++ ++ ++ ++++++++ * on none being left: at unmount every inode, and with it every ordered
++++++++++++ ++ ++ ++++++++ * extent, is gone before close_ctree() runs, delayed iputs are drained, and
++++++++++++ ++ ++ ++++++++ * on both paths the workers that complete ordered extents are stopped first.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_free_open_stripe_runs(struct btrfs_fs_info *fs_info)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ LIST_HEAD(bgs);
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg, *tmp_bg;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ list_splice_init(&fs_info->open_stripe_bgs, &bgs);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ list_for_each_entry_safe(bg, tmp_bg, &bgs, open_stripe_bg_list) {
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run, *tmp;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ list_del_init(&bg->open_stripe_bg_list);
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry_safe(run, tmp, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (run->inflight_bytes)
++++++++++++ ++ ++ ++++++++ btrfs_warn(fs_info,
++++++++++++ ++ ++ ++++++++"stripe run %llu-%llu in block group %llu released at teardown with %llu bytes still in flight",
++++++++++++ ++ ++ ++++++++ run->start, run->end, bg->start,
++++++++++++ ++ ++ ++++++++ run->inflight_bytes);
++++++++++++ ++ ++ ++++++++ list_del(&run->list);
++++++++++++ ++ ++ ++++++++ bitmap_free(run->live);
++++++++++++ ++ ++ ++++++++ bitmap_free(run->written);
++++++++++++ ++ ++ ++++++++ kfree(run);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ memset(bg->open_stripe, 0, sizeof(bg->open_stripe));
++++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder = 0;
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Close and drain the metadata stripe runs once every tree block this
++++++++++++ ++ ++ ++++++++ * transaction allocated has reached the disk.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * btrfs_retire_open_stripes() runs earlier in the commit and skips these
++++++++++++ ++ ++ ++++++++ * runs, so they stay open for the commit's own tree block allocations.
++++++++++++ ++ ++ ++++++++ * Closing earlier would strand blocks the commit has yet to allocate;
++++++++++++ ++ ++ ++++++++ * draining earlier would wait for writes the commit has yet to issue.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_retire_meta_stripes(struct btrfs_fs_info *fs_info, u64 transid)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ LIST_HEAD(retire_list);
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * No option test: a runtime disable leaves the running transaction's
++++++++++++ ++ ++ ++++++++ * metadata runs open, and the commit that follows still has to close
++++++++++++ ++ ++ ++++++++ * and drain them or their block groups never drop their references.
++++++++++++ ++ ++ ++++++++ * The list is empty when nothing is open, so this costs nothing on a
++++++++++++ ++ ++ ++++++++ * filesystem that never had the policy.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
++++++++++++ ++ ++ ++++++++ list_add_tail(&bg->open_stripe_retire_list, &retire_list);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ while (!list_empty(&retire_list)) {
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ bool again = true;
++++++++++++ ++ ++ ++++++++ bool warned = false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ bg = list_first_entry(&retire_list, struct btrfs_block_group,
++++++++++++ ++ ++ ++++++++ open_stripe_retire_list);
++++++++++++ ++ ++ ++++++++ list_del_init(&bg->open_stripe_retire_list);
++++++++++++ ++ ++ ++++++++ if (!(bg->flags & (BTRFS_BLOCK_GROUP_METADATA |
++++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ while (again) {
++++++++++++ ++ ++ ++++++++ u64 tail_start;
++++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ again = false;
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (!btrfs_stripe_run_class_is_meta(run->class) ||
++++++++++++ ++ ++ ++++++++ !run->open)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * The running transaction's runs are not ours
++++++++++++ ++ ++ ++++++++ * to close: their tree blocks are written, and
++++++++++++ ++ ++ ++++++++ * their stripes settled, by that transaction's
++++++++++++ ++ ++ ++++++++ * own commit.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (run->transid > transid)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ for (int band = 0;
++++++++++++ ++ ++ ++++++++ band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
++++++++++++ ++ ++ ++++++++ if (bg->open_stripe[BTRFS_STRIPE_RUN_META][band] == run)
++++++++++++ ++ ++ ++++++++ bg->open_stripe[BTRFS_STRIPE_RUN_META][band] = NULL;
++++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
++++++++++++ ++ ++ ++++++++ again = true;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (tail_len)
++++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * The closes above shrank the group's open-run remainder;
++++++++++++ ++ ++ ++++++++ * fold the delta into the space_info now. While the policy
++++++++++++ ++ ++ ++++++++ * is on the next allocation from this group would carry it,
++++++++++++ ++ ++ ++++++++ * but after a runtime disable nothing allocates here again
++++++++++++ ++ ++ ++++++++ * and the stale remainder holds metadata back for good.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Every tree block is on disk, so end_bbio_meta_write() has
++++++++++++ ++ ++ ++++++++ * reported all of them and this normally does not wait. A run
++++++++++++ ++ ++ ++++++++ * that does not drain has a tree block that was freed without
++++++++++++ ++ ++ ++++++++ * reporting to its run, an accounting bug: say which runs
++++++++++++ ++ ++ ++++++++ * once, and keep waiting. Only a filesystem error ends the
++++++++++++ ++ ++ ++++++++ * wait early.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ while (!wait_var_event_timeout(&bg->open_stripe_runs,
++++++++++++ ++ ++ ++++++++ !bg_has_undrained_meta_runs(bg, transid) ||
++++++++++++ ++ ++ ++++++++ BTRFS_FS_ERROR(fs_info), 60 * HZ)) {
++++++++++++ ++ ++ ++++++++ if (warned)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ btrfs_warn(fs_info,
++++++++++++ ++ ++ ++++++++"metadata stripe runs in block group %llu do not drain; a tree block was freed without reporting to its run",
++++++++++++ ++ ++ ++++++++ bg->start);
++++++++++++ ++ ++ ++++++++ btrfs_dump_open_stripe_runs(bg);
++++++++++++ ++ ++ ++++++++ warned = true;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Drained and no longer needed for padding or merging. */
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ while (true) {
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *next = NULL;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class) &&
++++++++++++ ++ ++ ++++++++ open_stripe_run_drained(run)) {
++++++++++++ ++ ++ ++++++++ next = run;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (!next)
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ free_open_stripe_run(bg, next);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Should this tree block be left dirty for the transaction commit to write?
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * A stripe of tree blocks goes out as one full-stripe write only if all of
++++++++++++ ++ ++ ++++++++ * its blocks are written in the same pass. Opportunistic writeback (memory
++++++++++++ ++ ++ ++++++++ * reclaim, the periodic timer) does not know about stripes and would split
++++++++++++ ++ ++ ++++++++ * it into read-modify-writes, so it is declined here, as zoned mode does in
++++++++++++ ++ ++ ++++++++ * btrfs_check_meta_write_pointer(), and a commit is requested instead.
++++++++++++ ++ ++ ++++++++ * WB_SYNC_ALL is never deferred, so a caller that needs the pages written
++++++++++++ ++ ++ ++++++++ * can always get them written.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++bool btrfs_defer_stripe_meta_write(struct btrfs_fs_info *fs_info,
++++++++++++ ++ ++ ++++++++ struct extent_buffer *eb,
++++++++++++ ++ ++ ++++++++ const struct writeback_control *wbc)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ bool defer;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ if (wbc->sync_mode == WB_SYNC_ALL)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, eb->start);
++++++++++++ ++ ++ ++++++++ if (!bg)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ defer = btrfs_is_stripe_meta_bg(bg);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (defer) {
++++++++++++ ++ ++ ++++++++ set_bit(BTRFS_FS_NEED_TRANS_COMMIT, &fs_info->flags);
++++++++++++ ++ ++ ++++++++ wake_up_process(fs_info->transaction_kthread);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ return defer;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Commit-time retirement: close every data stripe run opened before this
++++++++++++ ++ ++ ++++++++ * call and wait for its IO to complete. Metadata runs are retired later by
++++++++++++ ++ ++ ++++++++ * btrfs_retire_meta_stripes(), and persistent NOCOW runs survive. Must run
++++++++++++ ++ ++ ++++++++ * while the committing transaction still accepts joins (before
++++++++++++ ++ ++ ++++++++ * TRANS_STATE_COMMIT_DOING): tasks holding unreported run bytes can block on
++++++++++++ ++ ++ ++++++++ * a join, and this drain waits for their IO. Records the bumped sequence in
++++++++++++ ++ ++ ++++++++ * @trans; ordered extents from runs with open_seq >= it defer their file
++++++++++++ ++ ++ ++++++++ * extents to the next transaction. Calls are serialized by the transaction
++++++++++++ ++ ++ ++++++++ * commit; @trans is NULL for the final cleanup call.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info,
++++++++++++ ++ ++ ++++++++ struct btrfs_transaction *trans)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ LIST_HEAD(retire_list);
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ u64 seq;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ seq = ++fs_info->stripe_retire_seq;
++++++++++++ ++ ++ ++++++++ if (trans)
++++++++++++ ++ ++ ++++++++ trans->stripe_retire_seq = seq;
++++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
++++++++++++ ++ ++ ++++++++ list_add_tail(&bg->open_stripe_retire_list, &retire_list);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &retire_list, open_stripe_retire_list)
++++++++++++ ++ ++ ++++++++ close_block_group_stripe_runs(bg, seq);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Parked partial-stripe write rbios (see raid56.c) hold bios whose
++++++++++++ ++ ++ ++++++++ * write_done the drain below waits for; kick them all down first.
++++++++++++ ++ ++ ++++++++ * A racing park that saw its run still open lands after this flush
++++++++++++ ++ ++ ++++++++ * and is bounded by the park timer, not by us.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, 0, U64_MAX, false);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ while (!list_empty(&retire_list)) {
++++++++++++ ++ ++ ++++++++ bg = list_first_entry(&retire_list, struct btrfs_block_group,
++++++++++++ ++ ++ ++++++++ open_stripe_retire_list);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Same escape as btrfs_retire_block_group_stripes(): on an
++++++++++++ ++ ++ ++++++++ * errored filesystem the bytes in flight may never be
++++++++++++ ++ ++ ++++++++ * reported, and this wait sits in the transaction commit and
++++++++++++ ++ ++ ++++++++ * in the teardown call from close_ctree().
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ wait_var_event(&bg->open_stripe_runs,
++++++++++++ ++ ++ ++++++++ bg_open_stripes_settled(bg, seq) ||
++++++++++++ ++ ++ ++++++++ BTRFS_FS_ERROR(fs_info));
++++++++++++ ++ ++ ++++++++ if (BTRFS_FS_ERROR(fs_info) && !bg_open_stripes_settled(bg, seq))
++++++++++++ ++ ++ ++++++++ btrfs_warn(fs_info,
++++++++++++ ++ ++ ++++++++"block group %llu has stripe runs still in flight at retirement after a filesystem error; not waiting for them",
++++++++++++ ++ ++ ++++++++ bg->start);
++++++++++++ ++ ++ ++++++++ list_del_init(&bg->open_stripe_retire_list);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Drop the membership if the group has no runs left. */
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ if (list_empty(&bg->open_stripe_runs) &&
++++++++++++ ++ ++ ++++++++ !list_empty(&bg->open_stripe_bg_list)) {
++++++++++++ ++ ++ ++++++++ list_del_init(&bg->open_stripe_bg_list);
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ } else {
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Commit-time recompute of stripe_unusable for every stripe_alloc block group
++++++++++++ ++ ++ ++++++++ * whose free space changed this transaction. Trapped free space only settles
++++++++++++ ++ ++ ++++++++ * at commit -- partially filled stripes are returned by the retire above and
++++++++++++ ++ ++ ++++++++ * deleted extents are unpinned into the free space cache -- so this is where
++++++++++++ ++ ++ ++++++++ * the counter can be made authoritative. Called after btrfs_finish_extent_commit
++++++++++++ ++ ++ ++++++++ * so the free space cache reflects this transaction's frees.
++++++++++++ ++ ++ ++++++++ */
+++++++++++++++ +++++++++++static void reloc_reserve_resync(struct btrfs_block_group *bg);
+++++++++++++++ +++++++++++
++++++++++++ ++ ++ ++++++++void btrfs_scan_stripe_unusable(struct btrfs_fs_info *fs_info, bool force)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_space_info *sinfo;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ list_for_each_entry(sinfo, &fs_info->space_info, list) {
++++++++++++ ++ ++ ++++++++ u64 total = 0;
++++++++++++ ++ ++ ++++++++ u64 total_claimable = 0;
++++++++++++ ++ ++ ++++++++ u64 total_claimable_reloc = 0;
++++++++++++ ++ ++ ++++++++ u64 total_stranded = 0;
++++++++++++ ++ ++ ++++++++ u64 largest = 0;
++++++++++++ ++ ++ ++++++++ u64 unit = 0;
++++++++++++ ++ ++ ++++++++ u32 nr_rw = 0;
++++++++++++ ++ ++ ++++++++ int raid;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Data groups under stripe_alloc and metadata groups under
++++++++++++ ++ ++ ++++++++ * stripe_meta both place extents in whole stripes only, so
++++++++++++ ++ ++ ++++++++ * both carry trapped free space. Mixed groups are refused.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (!(sinfo->flags & (BTRFS_BLOCK_GROUP_DATA |
++++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_METADATA |
++++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ down_read(&sinfo->groups_sem);
++++++++++++ ++ ++ ++++++++ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!(btrfs_raid_array[raid].bg_flag &
++++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_RAID56_MASK))
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &sinfo->block_groups[raid],
++++++++++++ ++ ++ ++++++++ list) {
++++++++++++ ++ ++ ++++++++ bool dirty = test_and_clear_bit(
++++++++++++ ++ ++ ++++++++ BLOCK_GROUP_FLAG_STRIPE_UNUSABLE_DIRTY,
++++++++++++ ++ ++ ++++++++ &bg->runtime_flags);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * @force rescans every armed group regardless of
++++++++++++ ++ ++ ++++++++ * the dirty flag (debug: isolates a missed
++++++++++++ ++ ++ ++++++++ * dirty-marking from a genuine cache difference).
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (force || dirty)
++++++++++++ ++ ++ ++++++++ btrfs_block_group_rescan_stripe_unusable(bg);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Sum the authoritative per-group values into the space_info
++++++++++++ ++ ++ ++++++++ * total so it cannot drift from repeated arm/disarm/rescan
++++++++++++ ++ ++ ++++++++ * deltas. Disarmed (read-only) groups are excluded; their
++++++++++++ ++ ++ ++++++++ * free space is already accounted as read-only. Every update
++++++++++++ ++ ++ ++++++++ * of a group's values also updates the total under
++++++++++++ ++ ++ ++++++++ * sinfo->lock, so summing under it gives totals that match
++++++++++++ ++ ++ ++++++++ * the groups: claims, arms and disarms that race with the
++++++++++++ ++ ++ ++++++++ * scan are neither lost nor counted twice.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ spin_lock(&sinfo->lock);
++++++++++++ ++ ++ ++++++++ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!(btrfs_raid_array[raid].bg_flag &
++++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_RAID56_MASK))
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &sinfo->block_groups[raid],
++++++++++++ ++ ++ ++++++++ list) {
++++++++++++ ++ ++ ++++++++ if (bg->stripe_unusable_ready) {
++++++++++++ ++ ++ ++++++++ total += bg->stripe_unusable;
++++++++++++ ++ ++ ++++++++ total_stranded += READ_ONCE(bg->stripe_stranded);
++++++++++++ ++ ++ ++++++++ total_claimable += bg->stripe_claimable;
+++++++++++++++ +++++++++++ /* a held group's are in bytes_reloc_reserve */
+++++++++++++++ +++++++++++ if (bg->start == READ_ONCE(fs_info->data_reloc_bg) &&
+++++++++++++++ +++++++++++ !test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+++++++++++++++ +++++++++++ &bg->runtime_flags))
++++++++++++ ++ ++ ++++++++ total_claimable_reloc +=
++++++++++++ ++ ++ ++++++++ bg->stripe_claimable;
++++++++++++ ++ ++ ++++++++ nr_rw++;
++++++++++++ ++ ++ ++++++++ largest = max(largest, bg->length);
+++++++++++++++ +++++++++++ if (test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+++++++++++++++ +++++++++++ &bg->runtime_flags)) {
+++++++++++++++ +++++++++++ spin_lock(&bg->lock);
+++++++++++++++ +++++++++++ reloc_reserve_resync(bg);
+++++++++++++++ +++++++++++ spin_unlock(&bg->lock);
+++++++++++++++ +++++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ unit = max(unit, bg->full_stripe_len);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ up_read(&sinfo->groups_sem);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_unusable = total;
++++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_claimable = total_claimable;
++++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_claimable_reloc = total_claimable_reloc;
++++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_stranded = total_stranded;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Not if the option went off since the check above: the
++++++++++++ ++ ++ ++++++++ * disable path has cleared the margin and reserve, and no
++++++++++++ ++ ++ ++++++++ * later scan would clear them again.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (!(sinfo->flags & BTRFS_BLOCK_GROUP_DATA) &&
++++++++++++ ++ ++ ++++++++ btrfs_test_opt(fs_info, STRIPE_ALLOC)) {
++++++++++++ ++ ++ ++++++++ u64 reserve = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * stripe_meta. A tree block is reserved in bytes but
++++++++++++ ++ ++ ++++++++ * placed in whole stripes: the run it would join is
++++++++++++ ++ ++ ++++++++ * closed at the transaction boundary, so each group
++++++++++++ ++ ++ ++++++++ * claims at least one fresh stripe per transaction.
++++++++++++ ++ ++ ++++++++ * Pre-pay two transactions' worth of such claims, or
++++++++++++ ++ ++ ++++++++ * the last claimable stripes go to claims made for
++++++++++++ ++ ++ ++++++++ * reservations admitted earlier and an admitted tree
++++++++++++ ++ ++ ++++++++ * block finds none, which aborts the transaction.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_margin = 2ULL * nr_rw * unit;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * COW frees trap stripes (a stripe is claimable only
++++++++++++ ++ ++ ++++++++ * once every block in it is dead), and only relocation
++++++++++++ ++ ++ ++++++++ * packs live blocks back into whole stripes, which
++++++++++++ ++ ++ ++++++++ * needs room elsewhere for the moved group. With more
++++++++++++ ++ ++ ++++++++ * than one group, hold back one group's worth, capped
++++++++++++ ++ ++ ++++++++ * at a quarter of the space_info, from all but
++++++++++++ ++ ++ ++++++++ * relocation. Accounting only: a committing
++++++++++++ ++ ++ ++++++++ * transaction may still allocate here.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (nr_rw > 1 && unit)
++++++++++++ ++ ++ ++++++++ reserve = stripe_round_up(min(largest,
++++++++++++ ++ ++ ++++++++ sinfo->total_bytes / 4),
++++++++++++ ++ ++ ++++++++ unit);
++++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_reserve = reserve;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * The rescan usually lowers the counter (mid-transaction
++++++++++++ ++ ++ ++++++++ * incremental adds overcount conservatively); admission
++++++++++++ ++ ++ ++++++++ * tickets may be waiting on exactly that headroom.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ btrfs_try_granting_tickets(sinfo);
++++++++++++ ++ ++ ++++++++ spin_unlock(&sinfo->lock);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Does stripe-exclusive allocation force COW for an in-place (nocow or
++++++++++++ ++ ++ ++++++++ * prealloc) write at @bytenr? True for any raid56 data group under
++++++++++++ ++ ++ ++++++++ * stripe_alloc, since an in-place write could tear a stripe holding other
++++++++++++ ++ ++ ++++++++ * files' committed extents. The caller may still allow the write when
++++++++++++ ++ ++ ++++++++ * btrfs_stripe_nocow_writable() proves the stripes belong to the inode.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++bool btrfs_stripe_alloc_forces_cow(struct btrfs_fs_info *fs_info, u64 bytenr)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ bool ret = false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
++++++++++++ ++ ++ ++++++++ if (bg) {
++++++++++++ ++ ++ ++++++++ ret = btrfs_is_stripe_alloc_bg(bg);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ return ret;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Write-hole invariant checker, called for every raid56 write (full-stripe
++++++++++++ ++ ++ ++++++++ * and sub-stripe alike). Under stripe_alloc and stripe_meta a stripe may
++++++++++++ ++ ++ ++++++++ * only be written while an open or draining run covers it; a write outside
++++++++++++ ++ ++ ++++++++ * every run means a retired stripe is being rewritten or an allocation
++++++++++++ ++ ++ ++++++++ * bypassed the policy.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Groups that hosted RELOC or NOCOW runs are skipped: those extents are
++++++++++++ ++ ++ ++++++++ * legitimately overwritten in place after their runs drain.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_stripe_check_write(struct btrfs_fs_info *fs_info,
++++++++++++ ++ ++ ++++++++ u64 full_stripe_start, bool sub_stripe)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ unsigned long flags;
++++++++++++ ++ ++ ++++++++ bool live = false;
++++++++++++ ++ ++ ++++++++ u64 fsl;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ /* the legacy allocator's writes are still landing after a runtime enable */
++++++++++++ ++ ++ ++++++++ if (test_bit(BTRFS_FS_STRIPE_ALLOC_ENABLING, &fs_info->flags))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, full_stripe_start);
++++++++++++ ++ ++ ++++++++ if (!bg)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Metadata is checked on the same terms once stripe_meta places it:
++++++++++++ ++ ++ ++++++++ * its runs stay open across the commit's tree building but are closed
++++++++++++ ++ ++ ++++++++ * and drained before the commit ends, so a tree block write outside
++++++++++++ ++ ++ ++++++++ * every live run means a stripe was revisited after the transaction
++++++++++++ ++ ++ ++++++++ * that filled it finished -- the same violation as for data.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if ((!btrfs_is_stripe_alloc_bg(bg) && !btrfs_is_stripe_meta_bg(bg)) ||
++++++++++++ ++ ++ ++++++++ test_bit(BLOCK_GROUP_FLAG_STRIPE_RELOC_USED, &bg->runtime_flags) ||
++++++++++++ ++ ++ ++++++++ test_bit(BLOCK_GROUP_FLAG_STRIPE_NOCOW_USED, &bg->runtime_flags))
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ fsl = bg->full_stripe_len;
++++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
++++++++++++ ++ ++ ++++++++ if (full_stripe_start < run->end &&
++++++++++++ ++ ++ ++++++++ full_stripe_start + fsl > run->start) {
++++++++++++ ++ ++ ++++++++ live = true;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
++++++++++++ ++ ++ ++++++++ WARN_ONCE(!live,
++++++++++++ ++ ++ ++++++++"btrfs: %s write to stripe %llu (block group %llu) outside any live stripe run: write hole window violated",
++++++++++++ ++ ++ ++++++++ sub_stripe ? "sub-stripe" : "full-stripe",
++++++++++++ ++ ++ ++++++++ full_stripe_start, bg->start);
++++++++++++ ++ ++ ++++++++out:
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++#endif
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Drop the dedication of a block group to data relocation. Shared by the
++++++++++++ ++ ++ ++++++++ * zoned allocator and the stripe allocation policy; both dedicate one
++++++++++++ ++ ++ ++++++++ * block group at a time via fs_info->data_reloc_bg.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ bool cleared = false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->relocation_bg_lock);
++++++++++++ ++ ++ ++++++++ if (fs_info->data_reloc_bg == bg->start) {
++++++++++++ ++ ++ ++++++++ fs_info->data_reloc_bg = 0;
++++++++++++ ++ ++ ++++++++ cleared = true;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->relocation_bg_lock);
++++++++++++ ++ ++ ++++++++ if (cleared && bg->space_info) {
++++++++++++ ++ ++ ++++++++ spin_lock(&bg->space_info->lock);
++++++++++++ ++ ++ ++++++++ bg->space_info->bytes_stripe_claimable_reloc = 0;
++++++++++++ ++ ++ ++++++++ spin_unlock(&bg->space_info->lock);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Release the block group currently dedicated to data relocation, closing
++++++++++++ ++ ++ ++++++++ * its open stripe runs so ordinary allocations can use the remaining fully
++++++++++++ ++ ++ ++++++++ * free stripes. Used by the stripe allocation policy when relocation of a
++++++++++++ ++ ++ ++++++++ * block group finishes.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_release_data_reloc_bg(struct btrfs_fs_info *fs_info)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ u64 bytenr;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->relocation_bg_lock);
++++++++++++ ++ ++ ++++++++ bytenr = fs_info->data_reloc_bg;
++++++++++++ ++ ++ ++++++++ fs_info->data_reloc_bg = 0;
++++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->relocation_bg_lock);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!bytenr)
++++++++++++ ++ ++ ++++++++ goto release;
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
++++++++++++ ++ ++ ++++++++ if (!bg)
++++++++++++ ++ ++ ++++++++ goto release;
++++++++++++ ++ ++ ++++++++ if (bg->space_info) {
++++++++++++ ++ ++ ++++++++ spin_lock(&bg->space_info->lock);
++++++++++++ ++ ++ ++++++++ bg->space_info->bytes_stripe_claimable_reloc = 0;
++++++++++++ ++ ++ ++++++++ spin_unlock(&bg->space_info->lock);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ btrfs_close_bg_open_stripes(bg);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++release:
++++++++++++ ++ ++ ++++++++ btrfs_release_stripe_reloc_runs(fs_info);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
struct btrfs_caching_control *btrfs_get_caching_control(
struct btrfs_block_group *cache)
{
* btrfs_can_overcommit check here, and we need to pass in
* BTRFS_RESERVE_NO_FLUSH to give ourselves the most amount of
* leeway to allow us to mark this block group as read only.
- if (btrfs_can_overcommit(sinfo, num_bytes, BTRFS_RESERVE_NO_FLUSH))
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * As for data above: a stripe_meta group's trapped bytes are
++++++++++++ ++ ++ ++++++++ * already in the space_info total, do not re-provide them.
++++++++++++ + ++ ++++++++ */
- if (btrfs_can_overcommit(sinfo, num_bytes, BTRFS_RESERVE_NO_FLUSH))
++++++++++++ ++ ++ ++++++++ u64 trapped = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (READ_ONCE(cache->stripe_unusable_ready))
++++++++++++ ++ ++ ++++++++ trapped = min(READ_ONCE(cache->stripe_unusable), num_bytes);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * The stripe_meta reserve is the room a group's live blocks
++++++++++++ ++ ++ ++++++++ * move into when it is relocated, and going read-only is the
++++++++++++ ++ ++ ++++++++ * first step of exactly that: count the reserve as available
++++++++++++ ++ ++ ++++++++ * here, or the guard refuses every relocation at the edge by
++++++++++++ ++ ++ ++++++++ * the width of the reserve.
+++++++++++ ++ ++ ++++++++ */
----------- - -- -------- if (btrfs_can_overcommit(sinfo, num_bytes, BTRFS_RESERVE_NO_FLUSH))
++++++++++++ ++ ++ ++++++++ if (btrfs_can_overcommit(sinfo, num_bytes - trapped,
++++++++++++ ++ ++ ++++++++ BTRFS_RESERVE_NO_FLUSH, true))
++++++++++++ ++ ++ ++++++++ ret = 0;
+++++++++++++++ +++++++++++ /*
+++++++++++++++ +++++++++++ * Or the relocation reserve can take this group's live tree
+++++++++++++++ +++++++++++ * blocks: a failed tree block allocation outside the reserve
+++++++++++++++ +++++++++++ * falls back into it. Only with a reserve and something to
+++++++++++++++ +++++++++++ * move: an empty group would pass "0 >= 0" with the reserve
+++++++++++++++ +++++++++++ * off. Data gets no such exemption: its test above fails only
+++++++++++++++ +++++++++++ * while bytes_may_use depends on this group's free space, and
+++++++++++++++ +++++++++++ * data writeback has no reserve to fall back on.
+++ + */
+++++++++++++++ +++++++++++ else if (sinfo->bytes_reloc_reserve &&
+++++++++++++++ +++++++++++ cache->used + cache->reserved &&
+++++++++++++++ +++++++++++ sinfo->bytes_reloc_reserve >=
+++++++++++++++ +++++++++++ cache->used + cache->reserved)
+ + ret = 0;
}
if (!ret) {
list_add_tail(&cache->ro_list, &sinfo->ro_bgs);
}
out:
+++++++++++++++ +++++++++++ if (ret)
+++++++++++++++ +++++++++++ btrfs_reloc_reserve_rejoin(cache);
spin_unlock(&cache->lock);
spin_unlock(&sinfo->lock);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * A read-only group's free space (trapped included) is now accounted
++++++++++++ ++ ++ ++++++++ * as read-only and excluded from statfs; drop its stripe_unusable
++++++++++++ ++ ++ ++++++++ * contribution so it is not subtracted twice. Idempotent, so repeated
++++++++++++ ++ ++ ++++++++ * ro increments are harmless.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (!ret) {
++++++++++++ ++ ++ ++++++++ btrfs_block_group_disarm_stripe_unusable(cache);
++++++++++++ ++ ++ ++++++++ btrfs_stripe_unmanaged_sync(cache);
++++++++++++ ++ ++ ++++++++ }
if (ret == -ENOSPC && btrfs_test_opt(cache->fs_info, ENOSPC_DEBUG)) {
btrfs_info(cache->fs_info,
"unable to make block group %llu ro", cache->start);
spin_unlock(&bg->lock);
trace_btrfs_reclaim_block_group(bg);
----------- -- -- -------- ret = btrfs_relocate_chunk(fs_info, bg->start, false);
+++++++++++ ++ ++ ++++++++ btrfs_debug(fs_info,
- "reclaim: relocating block group %llu flags 0x%llx length %llu used %llu reserved %llu",
- bg->start, bg->flags,
- bg->length, used, reserved);
- ret = btrfs_relocate_chunk(fs_info, bg->start, false);
++++++++++++ ++ ++ ++++++++"reclaim: %s block group %llu flags 0x%llx length %llu used %llu reserved %llu trapped %llu",
++++++++++++ ++ ++ ++++++++ compact ? "compacting" : "relocating", bg->start, bg->flags,
++++++++++++ ++ ++ ++++++++ bg->length, used, reserved, trapped);
++++++++++++ ++ ++ ++++++++ if (compact) {
++++++++++++ ++ ++ ++++++++ ret = btrfs_compact_chunk(fs_info, bg->start, &compacted);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Compacted, the group stays: drop our read-only hold. It
++++++++++++ ++ ++ ++++++++ * gave back trapped space, not a chunk; what moved is counted
++++++++++++ ++ ++ ++++++++ * in stripe_run_stats.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (!ret && compacted) {
++++++++++++ ++ ++ ++++++++ btrfs_dec_block_group_ro(bg);
++++++++++++ ++ ++ ++++++++ used = 0;
++++++++++++ ++ ++ ++++++++ reserved = 0;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ } else {
++++++++++++ ++ ++ ++++++++ ret = btrfs_relocate_chunk(fs_info, bg->start, false);
++++++++++++ ++ ++ ++++++++ }
if (btrfs_is_zoned(fs_info) && ret == -EAGAIN) {
btrfs_dec_block_group_ro(bg);
btrfs_debug(fs_info, "deferring reclaim of chunk %llu", bg->start);
if (space_info->total_bytes < old_total)
btrfs_set_periodic_reclaim_ready(space_info, true);
spin_unlock(&space_info->lock);
----------- -- -- -------- if (!ret)
+++++++++++ ++ ++ ++++++++ if (!ret) {
(*reclaimed)++;
- btrfs_debug(fs_info, "reclaim: relocated block group %llu",
- bg->start);
++++++++++++ ++ ++ ++++++++ btrfs_debug(fs_info, "reclaim: %s block group %llu",
++++++++++++ ++ ++ ++++++++ compacted ? "compacted" : "relocated", bg->start);
+++++++++++ ++ ++ ++++++++ }
+
+ return ret;
+ }
+
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Reclaim gives up on a space_info for a while after this many reclaims in a
- * row failed with -ENOSPC. Relocating a group needs room for
++++++++++++ ++ ++ ++++++++ * row failed with -ENOSPC. Relocating or compacting a group needs room for
+++++++++++ ++ ++ ++++++++ * its live data (and metadata reservations for moving it); at the fill edge
+++++++++++ ++ ++ ++++++++ * there is none, every attempt fails after doing its work, and retrying each
+++++++++++ ++ ++ ++++++++ * failed group on every worker run held the balance exclusive operation for
+++++++++++ ++ ++ ++++++++ * over an hour while writers starved (octopus P1, bhive 2026-10-20).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++#define BTRFS_RECLAIM_ENOSPC_STREAK 3
+++++++++++ ++ ++ ++++++++#define BTRFS_RECLAIM_BACKOFF (10 * 60 * HZ)
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/* Free space that a retried reclaim could use: free in groups, unallocated. */
+++++++++++ ++ ++ ++++++++static u64 reclaim_free_bytes(const struct btrfs_space_info *space_info)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ return space_info->total_bytes - min(space_info->total_bytes,
+++++++++++ ++ ++ ++++++++ space_info->bytes_used) +
+++++++++++ ++ ++ ++++++++ atomic64_read(&space_info->fs_info->free_chunk_space);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Is reclaim backing off for this space_info? The backoff ends when its
+++++++++++ ++ ++ ++++++++ * time is up, when a chunk was added or removed, or when free space grew by
+++++++++++ ++ ++ ++++++++ * what the last failed group needed.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++bool btrfs_reclaim_backoff_active(struct btrfs_space_info *space_info)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ bool active = false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock(&space_info->lock);
+++++++++++ ++ ++ ++++++++ if (space_info->reclaim_backoff_until) {
+++++++++++ ++ ++ ++++++++ active = time_before(jiffies, space_info->reclaim_backoff_until) &&
+++++++++++ ++ ++ ++++++++ space_info->total_bytes == space_info->reclaim_backoff_total &&
+++++++++++ ++ ++ ++++++++ reclaim_free_bytes(space_info) < space_info->reclaim_backoff_free;
+++++++++++ ++ ++ ++++++++ if (!active) {
+++++++++++ ++ ++ ++++++++ space_info->reclaim_backoff_until = 0;
+++++++++++ ++ ++ ++++++++ space_info->reclaim_enospc_streak = 0;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock(&space_info->lock);
+++++++++++ ++ ++ ++++++++ return active;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Count a reclaim of @bg that failed with -ENOSPC (@enospc) or succeeded,
+++++++++++ ++ ++ ++++++++ * and start the backoff at the end of a streak of failures. @need is what
+++++++++++ ++ ++ ++++++++ * the group held: free space growing by that much ends the backoff early.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void reclaim_note_result(struct btrfs_block_group *bg, bool enospc, u64 need)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_space_info *space_info = bg->space_info;
+++++++++++ ++ ++ ++++++++ u32 streak;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock(&space_info->lock);
+++++++++++ ++ ++ ++++++++ if (!enospc) {
+++++++++++ ++ ++ ++++++++ space_info->reclaim_enospc_streak = 0;
+++++++++++ ++ ++ ++++++++ spin_unlock(&space_info->lock);
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ streak = ++space_info->reclaim_enospc_streak;
+++++++++++ ++ ++ ++++++++ if (streak >= BTRFS_RECLAIM_ENOSPC_STREAK) {
+++++++++++ ++ ++ ++++++++ space_info->reclaim_backoff_until = (jiffies + BTRFS_RECLAIM_BACKOFF) ?: 1;
+++++++++++ ++ ++ ++++++++ space_info->reclaim_backoff_total = space_info->total_bytes;
+++++++++++ ++ ++ ++++++++ space_info->reclaim_backoff_free = reclaim_free_bytes(space_info) +
+++++++++++ ++ ++ ++++++++ max_t(u64, need, SZ_16M);
+++++++++++ ++ ++ ++++++++ space_info->reclaim_backoffs++;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock(&space_info->lock);
+++++++++++ ++ ++ ++++++++ if (streak >= BTRFS_RECLAIM_ENOSPC_STREAK)
+++++++++++ ++ ++ ++++++++ btrfs_debug(bg->fs_info,
+++++++++++ ++ ++ ++++++++"reclaim: %u reclaims failed with ENOSPC (last block group %llu), backing off flags 0x%llx",
+++++++++++ ++ ++ ++++++++ streak, bg->start, space_info->flags);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
++++++++++++++++++ ++++++++/* A refused operation's retry window is open: see btrfs_exclop_start(). */
++++++++++++++++++ ++++++++static bool reclaim_yielding(const struct btrfs_fs_info *fs_info)
++++++++++++++++++ ++++++++{
++++++++++++++++++ ++++++++ const unsigned long until = READ_ONCE(fs_info->reclaim_yield_until);
++++++++++++++++++ ++++++++
++++++++++++++++++ ++++++++ return until && time_before(jiffies, until);
++++++++++++++++++ ++++++++}
+++++++++++++ ++++ ++++++++
- return ret;
++++++++++++++++++ ++++++++/*
++++++++++++++++++ ++++++++ * Unmounting, or read-only after an error: every group taken from here on is
++++++++++++++++++ ++++++++ * dropped (see btrfs_need_cleaner_sleep()), so do not wait out a gap for them.
++++++++++++++++++ ++++++++ */
++++++++++++++++++ ++++++++static bool reclaim_stopping(const struct btrfs_fs_info *fs_info)
++++++++++++++++++ ++++++++{
++++++++++++++++++ ++++++++ return btrfs_need_cleaner_sleep(fs_info) || BTRFS_FS_ERROR(fs_info);
+++++++++++++ ++++ ++++++++}
+++++++++++++ ++++ ++++++++
void btrfs_reclaim_block_groups(struct btrfs_fs_info *fs_info, unsigned int limit)
{
struct btrfs_block_group *bg;
LIST_HEAD(retry_list);
int reclaimed = 0;
- /* Relocation cannot proceed on an errored filesystem. */
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Relocation cannot proceed, and the stripe drain it starts with
++++++++++++ ++ ++ ++++++++ * would wait for IO that will not complete.
++++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (BTRFS_FS_ERROR(fs_info))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++
if (!btrfs_should_reclaim(fs_info))
return;
*/
list_sort(NULL, &fs_info->reclaim_bgs, reclaim_bgs_cmp);
while (!list_empty(&fs_info->reclaim_bgs)) {
+++++++++++ ++ ++ ++++++++ u64 need;
++++++++++++++++++ ++++++++ bool handoff = false;
int ret;
bg = list_first_entry(&fs_info->reclaim_bgs,
space_info = bg->space_info;
spin_unlock(&fs_info->unused_bgs_lock);
----------- -- -- -------- ret = btrfs_reclaim_block_group(bg, &reclaimed);
+++++++++++ ++ ++ ++++++++ need = READ_ONCE(bg->used);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Backing off: drop the group rather than requeue it. The
+++++++++++ ++ ++ ++++++++ * rescans and usage updates queue it again once reclaim may
- * succeed.
++++++++++++ ++ ++ ++++++++ * succeed. A stripe_reclaim_now request is tried regardless.
+++++++++++ ++ ++ ++++++++ */
- if (btrfs_reclaim_backoff_active(space_info)) {
++++++++++++ ++ ++ ++++++++ if (!READ_ONCE(bg->stripe_reclaim_forced) &&
++++++++++++ ++ ++ ++++++++ btrfs_reclaim_backoff_active(space_info)) {
+++++++++++ ++ ++ ++++++++ ret = 0;
+++++++++++ ++ ++ ++++++++ } else {
+++++++++++ ++ ++ ++++++++ int before = reclaimed;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ ret = btrfs_reclaim_block_group(bg, &reclaimed);
+++++++++++ ++ ++ ++++++++ if (ret == -ENOSPC || reclaimed > before)
+++++++++++ ++ ++ ++++++++ reclaim_note_result(bg, ret == -ENOSPC, need);
++++++++++++++++++ ++++++++ /*
++++++++++++++++++ ++++++++ * It was relocated, or failed to be (an error also
++++++++++++++++++ ++++++++ * comes from setting it read-only, which is quick).
++++++++++++++++++ ++++++++ * Once the limit is reached the loop ends below and
++++++++++++++++++ ++++++++ * releases the operation anyway: no gap.
++++++++++++++++++ ++++++++ */
++++++++++++++++++ ++++++++ handoff = (ret || reclaimed > before) && reclaimed < limit;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ /* An ENOSPC failure is not retried at once; see above. */
if ((btrfs_is_zoned(fs_info) && ret == -EAGAIN) ||
----------- -- -- -------- (ret && !READ_ONCE(space_info->periodic_reclaim)))
+++++++++++ ++ ++ ++++++++ (ret && ret != -ENOSPC &&
+++++++++++ ++ ++ ++++++++ !READ_ONCE(space_info->periodic_reclaim)))
btrfs_link_bg_list(bg, &retry_list);
btrfs_put_block_group(bg);
mutex_unlock(&fs_info->ro_block_group_mutex);
btrfs_end_transaction(trans);
- if (ret)
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * The group is read-only: close and drain its open stripe runs. An
++++++++++++ ++ ++ ++++++++ * allocation racing with the read-only transition either installed
++++++++++++ ++ ++ ++++++++ * its run before this (closed and drained here), or observes ->ro
++++++++++++ ++ ++ ++++++++ * under the stripe_run_lock and backs out.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (!ret)
++++++++++++ ++ ++ ++++++++ btrfs_retire_block_group_stripes(cache);
+++++++++++++++++++++++++++ else
+++++++++++++ +++++++++++++ btrfs_bg_unblock_nocow(cache);
return ret;
}
}
sinfo->bytes_readonly -= btrfs_block_group_available_space(cache);
list_del_init(&cache->ro_list);
++++++++++++ ++ ++ ++++++++ became_rw = true;
+++++++++++++++ +++++++++++ btrfs_reloc_reserve_rejoin(cache);
}
spin_unlock(&cache->lock);
spin_unlock(&sinfo->lock);
spin_unlock(&table->cache_lock);
}
++++++++++++++++++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
++++++++++++++++++ ++++++++/*
++++++++++++++++++ ++++++++ * Test knob (debug/rbio_cache_size): with a small or zero limit, a
++++++++++++++++++ ++++++++ * read-modify-write cannot take its stripe from the cache and must read it,
++++++++++++++++++ ++++++++ * which is what a fault-injecting test needs to see. Lowering it empties the
++++++++++++++++++ ++++++++ * cache, so the new limit holds from the next rbio on.
++++++++++++++++++ ++++++++ */
++++++++++++++++++ ++++++++void btrfs_set_rbio_cache_size(struct btrfs_fs_info *info, unsigned int size)
++++++++++++++++++ ++++++++{
++++++++++++++++++ ++++++++ unsigned int old = READ_ONCE(info->rbio_cache_size);
++++++++++++++++++ ++++++++
++++++++++++++++++ ++++++++ WRITE_ONCE(info->rbio_cache_size, size);
++++++++++++++++++ ++++++++ if (size < old && info->stripe_hash_table)
++++++++++++++++++ ++++++++ btrfs_clear_rbio_cache(info);
++++++++++++++++++ ++++++++}
++++++++++++++++++ ++++++++#endif
++++++++++++++++++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Send every parked rbio down and stop the park timer, before the raid56
++++++++++++ ++ ++ ++++++++ * workqueue is destroyed: the timer queues its expired rbios there. By
++++++++++++ ++ ++ ++++++++ * now nothing should be parked; a few passes cover an rbio that parks
++++++++++++ ++ ++ ++++++++ * again on its way down.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++void btrfs_stop_parked_rbios(struct btrfs_fs_info *fs_info)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_stripe_hash_table *table = fs_info->stripe_hash_table;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!table || !fs_info->rmw_workers)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ for (int i = 0; i < 3 && !list_empty_careful(&table->parked); i++) {
++++++++++++ ++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, 0, U64_MAX, true);
++++++++++++ ++ ++ ++++++++ flush_workqueue(fs_info->rmw_workers);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ cancel_delayed_work_sync(&table->parked_work);
++++++++++++ ++ ++ ++++++++ flush_workqueue(fs_info->rmw_workers);
++++++++++++ ++ ++ ++++++++ WARN_ON(!list_empty_careful(&table->parked));
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
/*
* remove all cached entries and free the hash table
* used by unmount
static void cache_rbio(struct btrfs_raid_bio *rbio)
{
struct btrfs_stripe_hash_table *table;
++++++++++++++++++ ++++++++ const unsigned int limit = rbio_cache_limit(rbio->bioc->fs_info);
++++++++++++ ++ ++ ++++++++ bool pin;
if (!test_bit(RBIO_CACHE_READY_BIT, &rbio->flags))
return;
- -
++++++++++++ ++ ++ ++++++++ pin = rbio_needs_pin(rbio);
++++++++++++++++++ ++++++++ /* Even a disabled cache keeps a stripe its run has not finished. */
++++++++++++++++++ ++++++++ if (!limit && !pin)
++++++++++++++++++ ++++++++ return;
+ +
table = rbio->bioc->fs_info->stripe_hash_table;
spin_lock(&table->cache_lock);
spin_unlock(&rbio->bio_list_lock);
------------ -- -- -------- if (table->cache_size > RBIO_CACHE_SIZE) {
------------ -- -- -------- struct btrfs_raid_bio *found;
------------ -- -- --------
------------ -- -- -------- found = list_last_entry(&table->stripe_cache,
------------ -- -- -------- struct btrfs_raid_bio,
------------ -- -- -------- stripe_cache);
------------ -- -- --------
------------ -- -- -------- if (found != rbio)
------------ -- -- -------- __remove_rbio_from_cache(found);
------------ -- -- -------- }
- - rbio_cache_prune(table, rbio, RBIO_CACHE_SIZE);
++++++++++++++++++ ++++++++ /* Prunes as far as needed: the limit can have been lowered. */
++++++++++++++++++ ++++++++ rbio_cache_prune(table, rbio, limit);
spin_unlock(&table->cache_lock);
}
}
}
---------- - -- -- --------/*
---------- - -- -- -------- * To determine if we need to read any sector from the disk.
---------- - -- -- -------- * Should only be utilized in RMW path, to skip cached rbio.
---------- - -- -- -------- */
---------- - -- -- --------static bool need_read_stripe_sectors(struct btrfs_raid_bio *rbio)
++++++++++++ ++ ++ ++++++++static void rbio_record_pad(struct btrfs_raid_bio *rbio,
++++++++++++ ++ ++ ++++++++ const struct btrfs_stripe_pad_info *info)
+ {
---------- - -- -- -------- int i;
++++++++++++ ++ ++ ++++++++ rbio->pad_reason = info->reason;
++++++++++++ ++ ++ ++++++++ rbio->pad_norun = info->norun;
++++++++++++ ++ ++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
++++++++++++ ++ ++ ++++++++ rbio->pad_run_open = info->run_open;
++++++++++++ ++ ++ ++++++++ rbio->pad_run_class = info->run_class;
++++++++++++ ++ ++ ++++++++ rbio->pad_run_start = info->run_start;
++++++++++++ ++ ++ ++++++++ rbio->pad_run_end = info->run_end;
++++++++++++ ++ ++ ++++++++ rbio->pad_run_offset = info->run_offset;
++++++++++++ ++ ++ ++++++++#endif
++++++++++++ ++ ++ ++++++++}
+
---------- - -- -- -------- for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) {
-------- - - -- -- -------- phys_addr_t paddr = rbio->stripe_paddrs[i * rbio->sector_nsteps];
++++++++++++ ++ ++ ++++++++static_assert(BTRFS_STRIPE_PAD_NR_REASONS + 1 ==
++++++++++++ ++ ++ ++++++++ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_park_stats.data_rmw_by));
++++++++++++ ++ ++ ++++++++static_assert(BTRFS_STRIPE_NORUN_NR ==
++++++++++++ ++ ++ ++++++++ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_park_stats.data_rmw_norun));
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
++++++++++++ ++ ++ ++++++++static_assert(BTRFS_STRIPE_PAD_NR_REASONS + 1 ==
++++++++++++ ++ ++ ++++++++ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_rmw_audit_stats.hole_by));
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/* What the audit found under one RMW's uncovered sectors. */
++++++++++++ ++ ++ ++++++++struct rmw_audit {
++++++++++++ ++ ++ ++++++++ int foreign; /* sectors under extents another root owns */
++++++++++++ ++ ++ ++++++++ int reloc; /* sectors under extents only relocation owns */
++++++++++++ ++ ++ ++++++++ u64 min_gen;
++++++++++++ ++ ++ ++++++++ u64 max_gen;
++++++++++++ ++ ++ ++++++++ u64 first; /* first foreign extent */
++++++++++++ ++ ++ ++++++++ u64 first_root; /* one other root referencing it (0: shared ref) */
++++++++++++ ++ ++ ++++++++};
+ +
-------- - - -- -- -------- /*
-------- - - -- -- -------- * We have a sector which doesn't have page nor uptodate,
-------- - - -- -- -------- * thus this rbio can not be cached one, as cached one must
-------- - - -- -- -------- * have all its data sectors present and uptodate.
-------- - - -- -- -------- */
-------- - - -- -- -------- if (paddr == INVALID_PADDR ||
-------- - - -- -- -------- !test_bit(i, rbio->stripe_uptodate_bitmap))
-------- - - -- -- -------- return true;
++++++++++++ ++ ++ ++++++++/* One extent being classified: its uncovered sectors and its owners. */
++++++++++++ ++ ++ ++++++++struct rmw_audit_ext {
++++++++++++ ++ ++ ++++++++ u64 bytenr;
++++++++++++ ++ ++ ++++++++ u64 gen;
++++++++++++ ++ ++ ++++++++ int sectors;
++++++++++++ ++ ++ ++++++++ int reloc_refs;
++++++++++++ ++ ++ ++++++++ int other_refs; /* includes shared refs: owner unknown, so foreign */
++++++++++++ ++ ++ ++++++++ u64 other_root;
++++++++++++ ++ ++ ++++++++};
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++static void rmw_audit_ext_ref(struct rmw_audit_ext *e, u64 root)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ if (root == BTRFS_DATA_RELOC_TREE_OBJECTID) {
++++++++++++ ++ ++ ++++++++ e->reloc_refs++;
++++++++++++ ++ ++ ++++++++ } else {
++++++++++++ ++ ++ ++++++++ e->other_refs++;
++++++++++++ ++ ++ ++++++++ if (!e->other_root)
++++++++++++ ++ ++ ++++++++ e->other_root = root;
+ + }
-------- - - -- -- -------- return false;
+ + }
+ +
-------- - - -- -- --------static void rmw_rbio(struct btrfs_raid_bio *rbio)
++++++++++++ ++ ++ ++++++++static void rmw_audit_ext_done(struct rmw_audit *a, struct rmw_audit_ext *e)
+ + {
-------- - - -- -- -------- struct bio_list bio_list;
-------- - - -- -- -------- int sectornr;
-------- - - -- -- -------- int ret = 0;
-------- - - -- -- --------
-------- - - -- -- -------- /*
-------- - - -- -- -------- * Allocate the pages for parity first, as P/Q pages will always be
-------- - - -- -- -------- * needed for both full-stripe and sub-stripe writes.
-------- - - -- -- -------- */
-------- - - -- -- -------- ret = alloc_rbio_parity_pages(rbio);
-------- - - -- -- -------- if (ret < 0)
++++++++++++ ++ ++ ++++++++ if (!e->sectors)
+ + goto out;
++++++++++++ ++ ++ ++++++++ /* No ref seen at all: be conservative and call it foreign. */
++++++++++++ ++ ++ ++++++++ if (e->other_refs || !e->reloc_refs) {
++++++++++++ ++ ++ ++++++++ if (!a->foreign) {
++++++++++++ ++ ++ ++++++++ a->first = e->bytenr;
++++++++++++ ++ ++ ++++++++ a->first_root = e->other_root;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ a->foreign += e->sectors;
++++++++++++ ++ ++ ++++++++ a->min_gen = min(a->min_gen, e->gen);
++++++++++++ ++ ++ ++++++++ a->max_gen = max(a->max_gen, e->gen);
++++++++++++ ++ ++ ++++++++ } else {
++++++++++++ ++ ++ ++++++++ a->reloc += e->sectors;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++out:
++++++++++++ ++ ++ ++++++++ memset(e, 0, sizeof(*e));
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Classify the data sectors of this rbio's stripe that the write does NOT
++++++++++++ ++ ++ ++++++++ * cover by the COMMITTED extent under them and that extent's owners. Those
++++++++++++ ++ ++ ++++++++ * are the sectors a torn RMW can destroy after a crash. An extent owned only
++++++++++++ ++ ++ ++++++++ * by the data reloc tree is relocation's own orphaned copy, discarded by
++++++++++++ ++ ++ ++++++++ * orphan cleanup if a crash comes before its pointers move; any other owner
++++++++++++ ++ ++ ++++++++ * is a real write hole. Commit root, no locking, as in
++++++++++++ ++ ++ ++++++++ * rmw_block_referenced(). Returns -1 when the lookup cannot answer.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * The commit root is switched shortly before the super lands, so an extent
++++++++++++ ++ ++ ++++++++ * of the transaction committing right now can count early: this errs towards
++++++++++++ ++ ++ ++++++++ * reporting a hole, never towards hiding one.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static int rmw_committed_uncovered(struct btrfs_raid_bio *rbio,
++++++++++++ ++ ++ ++++++++ struct rmw_audit *a)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
++++++++++++ ++ ++ ++++++++ const u64 stripe_start = rbio->bioc->full_stripe_logical;
++++++++++++ ++ ++ ++++++++ const u64 stripe_end = stripe_start +
++++++++++++ ++ ++ ++++++++ (u64)rbio->nr_data * BTRFS_STRIPE_LEN;
++++++++++++ ++ ++ ++++++++ const u32 bits = fs_info->sectorsize_bits;
++++++++++++ ++ ++ ++++++++ struct btrfs_root *extent_root = btrfs_extent_root(fs_info, stripe_start);
++++++++++++ ++ ++ ++++++++ struct rmw_audit_ext e = { 0 };
++++++++++++ ++ ++ ++++++++ struct btrfs_path *path;
++++++++++++ ++ ++ ++++++++ struct btrfs_key key;
++++++++++++ ++ ++ ++++++++ int ret;
+ +
++++++++++++ ++ ++ ++++++++ memset(a, 0, sizeof(*a));
++++++++++++ ++ ++ ++++++++ a->min_gen = U64_MAX;
+ + /*
-------- - - -- -- -------- * Either full stripe write, or we have every data sector already
-------- - - -- -- -------- * cached, can go to write path immediately.
++++++++++++ ++ ++ ++++++++ * The caller holds the full stripe lock, so nothing here may wait for
++++++++++++ ++ ++ ++++++++ * something that can wait for this stripe: a tree block read that
++++++++++++ ++ ++ ++++++++ * needs a raid56 rebuild of it (mixed block groups keep tree blocks in
++++++++++++ ++ ++ ++++++++ * it, see fill_data_csums(), so they are not audited), or
++++++++++++ ++ ++ ++++++++ * commit_root_sem, whose holder may be reading such a block. Read
++++++++++++ ++ ++ ++++++++ * cached blocks only, and take the semaphore only if it is free. Each
++++++++++++ ++ ++ ++++++++ * miss counts as an audit error.
+ + */
-------- - - -- -- -------- if (!rbio_is_full(rbio) && need_read_stripe_sectors(rbio)) {
-------- - - -- -- -------- /*
-------- - - -- -- -------- * Now we're doing sub-stripe write, also need all data stripes
-------- - - -- -- -------- * to do the full RMW.
-------- - - -- -- -------- */
-------- - - -- -- -------- ret = alloc_rbio_data_pages(rbio);
-------- - - -- -- -------- if (ret < 0)
-------- - - -- -- -------- goto out;
-------- - - -- -- --------
-------- - - -- -- -------- index_rbio_pages(rbio);
++++++++++++ ++ ++ ++++++++ if (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_METADATA)
++++++++++++ ++ ++ ++++++++ return -1;
++++++++++++ ++ ++ ++++++++ path = btrfs_alloc_path();
++++++++++++ ++ ++ ++++++++ if (!path)
++++++++++++ ++ ++ ++++++++ return -1;
++++++++++++ ++ ++ ++++++++ path->search_commit_root = 1;
++++++++++++ ++ ++ ++++++++ path->skip_locking = 1;
++++++++++++ ++ ++ ++++++++ path->nowait = 1;
+ +
-------- - - -- -- -------- ret = rmw_read_wait_recover(rbio);
++++++++++++ ++ ++ ++++++++ if (!down_read_trylock(&fs_info->commit_root_sem)) {
++++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
++++++++++++ ++ ++ ++++++++ return -1;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ key.objectid = stripe_start;
++++++++++++ ++ ++ ++++++++ key.type = BTRFS_EXTENT_ITEM_KEY;
++++++++++++ ++ ++ ++++++++ key.offset = 0;
++++++++++++ ++ ++ ++++++++ ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
++++++++++++ ++ ++ ++++++++ if (ret < 0)
++++++++++++ ++ ++ ++++++++ goto fail;
++++++++++++ ++ ++ ++++++++ /* An extent starting below the stripe may reach into it. */
++++++++++++ ++ ++ ++++++++ ret = btrfs_previous_extent_item(extent_root, path,
++++++++++++ ++ ++ ++++++++ stripe_start > SZ_1G ? stripe_start - SZ_1G : 0);
++++++++++++ ++ ++ ++++++++ if (ret < 0)
++++++++++++ ++ ++ ++++++++ goto fail;
++++++++++++ ++ ++ ++++++++ if (ret > 0) {
++++++++++++ ++ ++ ++++++++ btrfs_release_path(path);
++++++++++++ ++ ++ ++++++++ ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+ + if (ret < 0)
-------- - - -- -- -------- goto out;
++++++++++++ ++ ++ ++++++++ goto fail;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ while (1) {
++++++++++++ ++ ++ ++++++++ struct extent_buffer *leaf = path->nodes[0];
++++++++++++ ++ ++ ++++++++ const int slot = path->slots[0];
++++++++++++ ++ ++ ++++++++ struct btrfs_extent_item *ei;
++++++++++++ ++ ++ ++++++++ unsigned long ptr, end;
++++++++++++ ++ ++ ++++++++ u64 lo, hi;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (slot >= btrfs_header_nritems(leaf)) {
++++++++++++ ++ ++ ++++++++ ret = btrfs_next_leaf(extent_root, path);
++++++++++++ ++ ++ ++++++++ if (ret < 0)
++++++++++++ ++ ++ ++++++++ goto fail;
++++++++++++ ++ ++ ++++++++ if (ret > 0)
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ btrfs_item_key_to_cpu(leaf, &key, slot);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Keyed backrefs of the extent being classified. */
++++++++++++ ++ ++ ++++++++ if (e.sectors && key.objectid == e.bytenr) {
++++++++++++ ++ ++ ++++++++ if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
++++++++++++ ++ ++ ++++++++ struct btrfs_extent_data_ref *dref;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ dref = btrfs_item_ptr(leaf, slot,
++++++++++++ ++ ++ ++++++++ struct btrfs_extent_data_ref);
++++++++++++ ++ ++ ++++++++ rmw_audit_ext_ref(&e, btrfs_extent_data_ref_root(leaf, dref));
++++++++++++ ++ ++ ++++++++ } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
++++++++++++ ++ ++ ++++++++ e.other_refs++;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ path->slots[0]++;
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (key.type != BTRFS_EXTENT_ITEM_KEY) {
++++++++++++ ++ ++ ++++++++ if (key.objectid >= stripe_end)
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ path->slots[0]++;
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ rmw_audit_ext_done(a, &e);
++++++++++++ ++ ++ ++++++++ if (key.objectid >= stripe_end)
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ if (key.objectid + key.offset <= stripe_start) {
++++++++++++ ++ ++ ++++++++ path->slots[0]++;
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item);
++++++++++++ ++ ++ ++++++++ lo = max(key.objectid, stripe_start);
++++++++++++ ++ ++ ++++++++ hi = min(key.objectid + key.offset, stripe_end);
++++++++++++ ++ ++ ++++++++ for (u64 b = lo; b < hi; b += fs_info->sectorsize)
++++++++++++ ++ ++ ++++++++ if (rbio->bio_paddrs[((b - stripe_start) >> bits) * rbio->sector_nsteps] == INVALID_PADDR)
++++++++++++ ++ ++ ++++++++ e.sectors++;
++++++++++++ ++ ++ ++++++++ if (!e.sectors) {
++++++++++++ ++ ++ ++++++++ path->slots[0]++;
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ e.bytenr = key.objectid;
++++++++++++ ++ ++ ++++++++ e.gen = btrfs_extent_generation(leaf, ei);
++++++++++++ ++ ++ ++++++++ /* Inline backrefs: data extents carry no tree_block_info. */
++++++++++++ ++ ++ ++++++++ ptr = (unsigned long)(ei + 1);
++++++++++++ ++ ++ ++++++++ end = (unsigned long)ei + btrfs_item_size(leaf, slot);
++++++++++++ ++ ++ ++++++++ while (ptr < end) {
++++++++++++ ++ ++ ++++++++ struct btrfs_extent_inline_ref *iref = (void *)ptr;
++++++++++++ ++ ++ ++++++++ const int type = btrfs_extent_inline_ref_type(leaf, iref);
++++++++++++ ++ ++ ++++++++ const u32 size = btrfs_extent_inline_ref_size(type);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (type == BTRFS_EXTENT_DATA_REF_KEY) {
++++++++++++ ++ ++ ++++++++ struct btrfs_extent_data_ref *dref =
++++++++++++ ++ ++ ++++++++ (void *)&iref->offset;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ rmw_audit_ext_ref(&e, btrfs_extent_data_ref_root(leaf, dref));
++++++++++++ ++ ++ ++++++++ } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
++++++++++++ ++ ++ ++++++++ e.other_refs++;
++++++++++++ ++ ++ ++++++++ } else if (type != BTRFS_EXTENT_OWNER_REF_KEY) {
++++++++++++ ++ ++ ++++++++ /* Not a data ref we understand: foreign. */
++++++++++++ ++ ++ ++++++++ e.other_refs++;
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (!size)
++++++++++++ ++ ++ ++++++++ break;
++++++++++++ ++ ++ ++++++++ ptr += size;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ path->slots[0]++;
+ + }
++++++++++++ ++ ++ ++++++++ rmw_audit_ext_done(a, &e);
++++++++++++ ++ ++ ++++++++ up_read(&fs_info->commit_root_sem);
++++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
++++++++++++ ++ ++ ++++++++ return a->foreign + a->reloc;
++++++++++++ ++ ++ ++++++++fail:
++++++++++++ ++ ++ ++++++++ up_read(&fs_info->commit_root_sem);
++++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
++++++++++++ ++ ++ ++++++++ return -1;
++++++++++++ ++ ++ ++++++++}
+ +
-------- - - -- -- -------- /*
-------- - - -- -- -------- * At this stage we're not allowed to add any new bios to the
-------- - - -- -- -------- * bio list any more, anyone else that wants to change this stripe
-------- - - -- -- -------- * needs to do their own rmw.
-------- - - -- -- -------- */
-------- - - -- -- -------- spin_lock(&rbio->bio_list_lock);
-------- - - -- -- -------- set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
-------- - - -- -- -------- spin_unlock(&rbio->bio_list_lock);
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * With /sys/fs/btrfs/<uuid>/debug/stripe_rmw_audit set: prove from the
++++++++++++ ++ ++ ++++++++ * committed extent tree, not from the allocator's design, that a data RMW put
++++++++++++ ++ ++ ++++++++ * no other owner's committed data at risk, and log the writer of any that
++++++++++++ ++ ++ ++++++++ * did (and of every NO_RUN RMW, the class the allocator cannot explain by
++++++++++++ ++ ++ ++++++++ * construction). @idx is the RMW's data_rmw_by index.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static void rbio_audit_rmw(struct btrfs_raid_bio *rbio, u8 idx)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
++++++++++++ ++ ++ ++++++++ const u64 stripe_start = rbio->bioc->full_stripe_logical;
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ struct rmw_audit a;
++++++++++++ ++ ++ ++++++++ u64 root_id = 0, ino = 0, foff = 0, oflags = 0, oseq = 0, odisk = 0;
++++++++++++ ++ ++ ++++++++ u64 bg_flags = 0;
++++++++++++ ++ ++ ++++++++ bool orun = false;
++++++++++++ ++ ++ ++++++++ int nbios = 0;
++++++++++++ ++ ++ ++++++++ int ro = -1;
++++++++++++ ++ ++ ++++++++ int ret;
++++++++++++ ++ ++ ++++++++ struct bio *bio;
+ +
-------- - - -- -- -------- bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors);
++++++++++++ ++ ++ ++++++++ if (!READ_ONCE(fs_info->stripe_rmw_audit))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.audited);
+ +
-------- - - -- -- -------- index_rbio_pages(rbio);
++++++++++++ ++ ++ ++++++++ spin_lock(&rbio->bio_list_lock);
++++++++++++ ++ ++ ++++++++ bio_list_for_each(bio, &rbio->bio_list) {
++++++++++++ ++ ++ ++++++++ struct btrfs_bio *bbio = btrfs_bio(bio);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (nbios++)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (!bbio->inode ||
++++++++++++ ++ ++ ++++++++ btrfs_ino(bbio->inode) == BTRFS_BTREE_INODE_OBJECTID)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ root_id = btrfs_root_id(bbio->inode->root);
++++++++++++ ++ ++ ++++++++ ino = btrfs_ino(bbio->inode);
++++++++++++ ++ ++ ++++++++ foff = bbio->file_offset;
++++++++++++ ++ ++ ++++++++ if (bbio->ordered) {
++++++++++++ ++ ++ ++++++++ oflags = bbio->ordered->flags;
++++++++++++ ++ ++ ++++++++ oseq = bbio->ordered->stripe_seq;
++++++++++++ ++ ++ ++++++++ odisk = bbio->ordered->disk_bytenr;
++++++++++++ ++ ++ ++++++++ orun = bbio->ordered->stripe_run != NULL;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ spin_unlock(&rbio->bio_list_lock);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* The audit needs bio sectors mapped; idempotent. */
++++++++++++ ++ ++ ++++++++ index_rbio_pages(rbio);
++++++++++++ ++ ++ ++++++++ ret = rmw_committed_uncovered(rbio, &a);
++++++++++++ ++ ++ ++++++++ if (ret < 0) {
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.audit_err);
++++++++++++ ++ ++ ++++++++ } else if (a.foreign) {
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.hole_by[idx]);
++++++++++++ ++ ++ ++++++++ atomic64_add(a.foreign, &fs_info->stripe_rmw_audit_stats.hole_sectors);
++++++++++++ ++ ++ ++++++++ if (root_id == BTRFS_DATA_RELOC_TREE_OBJECTID)
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.reloc_writer_foreign);
++++++++++++ ++ ++ ++++++++ } else if (a.reloc) {
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.hole_reloc_only);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (ret > 0)
++++++++++++ ++ ++ ++++++++ atomic64_add(a.reloc, &fs_info->stripe_rmw_audit_stats.hole_reloc_sectors);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (idx == BTRFS_STRIPE_PAD_NO_RUN) {
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
++++++++++++ ++ ++ ++++++++ if (bg) {
++++++++++++ ++ ++ ++++++++ ro = data_race(bg->ro);
++++++++++++ ++ ++ ++++++++ bg_flags = bg->flags;
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Separate call sites, so separate rate limits: a flood of NO_RUN
++++++++++++ ++ ++ ++++++++ * lines cannot starve the hole lines.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (ret > 0 && a.foreign)
++++++++++++ ++ ++ ++++++++ btrfs_warn_rl(fs_info,
++++++++++++ ++ ++ ++++++++"rmw audit: write hole: stripe %llu reason %u norun %u foreign %d reloc %d gen %llu..%llu first %llu first_root %lld fsgen %llu committed %llu writer root %lld ino %llu foff %llu oe_flags 0x%llx oe_disk %llu oe_seq %llu oe_run %d nbios %d",
++++++++++++ ++ ++ ++++++++ stripe_start, idx, rbio->pad_norun, a.foreign,
++++++++++++ ++ ++ ++++++++ a.reloc, a.min_gen, a.max_gen, a.first,
++++++++++++ ++ ++ ++++++++ (s64)a.first_root, btrfs_get_fs_generation(fs_info),
++++++++++++ ++ ++ ++++++++ btrfs_get_last_trans_committed(fs_info),
++++++++++++ ++ ++ ++++++++ (s64)root_id, ino, foff, oflags, odisk, oseq, orun,
++++++++++++ ++ ++ ++++++++ nbios);
++++++++++++ ++ ++ ++++++++ else if (idx == BTRFS_STRIPE_PAD_NO_RUN)
++++++++++++ ++ ++ ++++++++ btrfs_info_rl(fs_info,
++++++++++++ ++ ++ ++++++++"rmw audit: no_run RMW stripe %llu norun %u bg_ro %d bg_flags 0x%llx run %llu..%llu off %llu open %u class %u reloc_sectors %d fsgen %llu committed %llu writer root %lld ino %llu foff %llu oe_flags 0x%llx oe_seq %llu oe_run %d nbios %d",
++++++++++++ ++ ++ ++++++++ stripe_start, rbio->pad_norun, ro, bg_flags,
++++++++++++ ++ ++ ++++++++ rbio->pad_run_start, rbio->pad_run_end,
++++++++++++ ++ ++ ++++++++ rbio->pad_run_offset, rbio->pad_run_open,
++++++++++++ ++ ++ ++++++++ rbio->pad_run_class, ret > 0 ? a.reloc : 0,
++++++++++++ ++ ++ ++++++++ btrfs_get_fs_generation(fs_info),
++++++++++++ ++ ++ ++++++++ btrfs_get_last_trans_committed(fs_info),
++++++++++++ ++ ++ ++++++++ (s64)root_id, ino, foff, oflags, oseq, orun, nbios);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++#else
++++++++++++ ++ ++ ++++++++static void rbio_audit_rmw(struct btrfs_raid_bio *rbio, u8 idx) { }
++++++++++++ ++ ++ ++++++++#endif
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Attribute a read-modify-write to why padding declined for this rbio. With
++++++++++++ ++ ++ ++++++++ * allow_rmw empty, a data RMW in a stripe_alloc group is expected in two
++++++++++++ ++ ++ ++++++++ * places only: at the open run's own frontier (UNCOVERED: the sectors it
++++++++++++ ++ ++ ++++++++ * reads are the same run's, in the same transaction, not yet durable), and
++++++++++++ ++ ++ ++++++++ * from relocation writing into its own preallocated extents, which nothing
++++++++++++ ++ ++ ++++++++ * but the data reloc inode references until relocation moves the pointers
++++++++++++ ++ ++ ++++++++ * after writeback. Any other data RMW in a covered group is unexplained.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static void rbio_count_rmw_reason(struct btrfs_raid_bio *rbio)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
++++++++++++ ++ ++ ++++++++ u8 idx = rbio->pad_reason;
++++++++++++ ++ ++ ++++++++ bool reloc_writer = false;
++++++++++++ ++ ++ ++++++++ struct bio *bio;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (idx >= BTRFS_STRIPE_PAD_NR_REASONS)
++++++++++++ ++ ++ ++++++++ idx = BTRFS_STRIPE_PAD_NR_REASONS; /* not tried */
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_by[idx]);
++++++++++++ ++ ++ ++++++++ if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (idx == BTRFS_STRIPE_PAD_NO_RUN) {
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (rbio->pad_norun < BTRFS_STRIPE_NORUN_NR)
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_norun[rbio->pad_norun]);
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info,
++++++++++++ ++ ++ ++++++++ rbio->bioc->full_stripe_logical);
++++++++++++ ++ ++ ++++++++ if (bg) {
++++++++++++ ++ ++ ++++++++ if (data_race(bg->ro))
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_norun_ro);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* The writer: the first bio's inode. */
++++++++++++ ++ ++ ++++++++ spin_lock(&rbio->bio_list_lock);
++++++++++++ ++ ++ ++++++++ bio = bio_list_peek(&rbio->bio_list);
++++++++++++ ++ ++ ++++++++ if (bio && btrfs_bio(bio)->inode)
++++++++++++ ++ ++ ++++++++ reloc_writer = btrfs_is_data_reloc_root(btrfs_bio(bio)->inode->root);
++++++++++++ ++ ++ ++++++++ spin_unlock(&rbio->bio_list_lock);
++++++++++++ ++ ++ ++++++++ /* Disjoint from the frontier count, so the two can be summed. */
++++++++++++ ++ ++ ++++++++ if (reloc_writer && idx != BTRFS_STRIPE_PAD_UNCOVERED)
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_reloc_writer);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ rbio_audit_rmw(rbio, idx);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Try to avoid the RMW read phase of a sub-stripe write by zero-filling the
++++++++++++ ++ ++ ++++++++ * uncovered data sectors. Legal only where the stripe's run says a sector
++++++++++++ ++ ++ ++++++++ * is dead: never allocated, handed back, or allocated with its write still
++++++++++++ ++ ++ ++++++++ * to come. That write, like any allocation the run makes after this check,
++++++++++++ ++ ++ ++++++++ * serializes behind this rbio's stripe lock and lands over the zeros.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static bool rmw_try_pad_full(struct btrfs_raid_bio *rbio)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
++++++++++++ ++ ++ ++++++++ const u64 stripe_start = rbio->bioc->full_stripe_logical;
++++++++++++ ++ ++ ++++++++ const u64 stripe_len = (u64)rbio->nr_data * BTRFS_STRIPE_LEN;
++++++++++++ ++ ++ ++++++++ const u32 sectorsize = fs_info->sectorsize;
++++++++++++ ++ ++ ++++++++ const u32 nsectors = rbio->nr_data * rbio->stripe_nsectors;
++++++++++++ ++ ++ ++++++++ struct btrfs_stripe_pad_info info;
++++++++++++ ++ ++ ++++++++ unsigned long *dead;
++++++++++++ ++ ++ ++++++++ unsigned long *unwritten;
++++++++++++ ++ ++ ++++++++ u32 covered = 0;
++++++++++++ ++ ++ ++++++++ u32 nr_unwritten = 0;
++++++++++++ ++ ++ ++++++++ bool ok = false;
++++++++++++ ++ ++ ++++++++ int i;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ rbio->pad_reason = RBIO_PAD_NOT_TRIED;
++++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ dead = bitmap_zalloc(nsectors, GFP_NOFS);
++++++++++++ ++ ++ ++++++++ unwritten = bitmap_zalloc(nsectors, GFP_NOFS);
++++++++++++ ++ ++ ++++++++ if (!dead || !unwritten) {
++++++++++++ ++ ++ ++++++++ bitmap_free(dead);
++++++++++++ ++ ++ ++++++++ bitmap_free(unwritten);
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (!btrfs_stripe_run_pad_mask(fs_info, stripe_start, stripe_len, dead,
++++++++++++ ++ ++ ++++++++ unwritten, &info)) {
++++++++++++ ++ ++ ++++++++ rbio_record_pad(rbio, &info);
++++++++++++ ++ ++ ++++++++ trace_btrfs_stripe_pad_decline(fs_info, stripe_start,
++++++++++++ ++ ++ ++++++++ stripe_len, 0, 0, nsectors,
++++++++++++ ++ ++ ++++++++ info.run_start, info.run_end,
++++++++++++ ++ ++ ++++++++ info.run_offset,
++++++++++++ ++ ++ ++++++++ info.inflight_bytes,
++++++++++++ ++ ++ ++++++++ info.reason);
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Populate bio_paddrs so covered sectors are visible below. */
++++++++++++ ++ ++ ++++++++ index_rbio_pages(rbio);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ for (i = 0; i < nsectors; i++)
++++++++++++ ++ ++ ++++++++ if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
++++++++++++ ++ ++ ++++++++ covered++;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Every uncovered sector must be dead per the run. A live uncovered
++++++++++++ ++ ++ ++++++++ * sector that was already written holds data this write does not
++++++++++++ ++ ++ ++++++++ * carry, so refuse (UNCOVERED); the stripe pin (rbio_needs_pin())
++++++++++++ ++ ++ ++++++++ * normally supplies such sectors from the cache, so nothing is read.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ for (i = 0; i < nsectors; i++) {
++++++++++++ ++ ++ ++++++++ if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (!test_bit(i, dead)) {
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_uncovered);
++++++++++++ ++ ++ ++++++++ info.reason = BTRFS_STRIPE_PAD_UNCOVERED;
++++++++++++ ++ ++ ++++++++ rbio_record_pad(rbio, &info);
++++++++++++ ++ ++ ++++++++ trace_btrfs_stripe_pad_decline(fs_info, stripe_start,
++++++++++++ ++ ++ ++++++++ stripe_len, 0, covered,
++++++++++++ ++ ++ ++++++++ nsectors,
++++++++++++ ++ ++ ++++++++ info.run_start,
++++++++++++ ++ ++ ++++++++ info.run_end,
++++++++++++ ++ ++ ++++++++ info.run_offset,
++++++++++++ ++ ++ ++++++++ info.inflight_bytes,
++++++++++++ ++ ++ ++++++++ info.reason);
++++++++++++ ++ ++ ++++++++ goto out;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (test_bit(i, unwritten))
++++++++++++ ++ ++ ++++++++ nr_unwritten++;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ ok = true;
++++++++++++ ++ ++ ++++++++out:
++++++++++++ ++ ++ ++++++++ bitmap_free(dead);
++++++++++++ ++ ++ ++++++++ bitmap_free(unwritten);
++++++++++++ ++ ++ ++++++++ if (!ok)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* The oracle agreed: only a page allocation failure refuses now. */
++++++++++++ ++ ++ ++++++++ rbio->pad_reason = BTRFS_STRIPE_PAD_OK;
++++++++++++ ++ ++ ++++++++ if (alloc_rbio_data_pages(rbio) < 0)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) {
++++++++++++ ++ ++ ++++++++ const int stripe_nr = i / rbio->stripe_nsectors;
++++++++++++ ++ ++ ++++++++ const int sectornr = i % rbio->stripe_nsectors;
++++++++++++ ++ ++ ++++++++ int step;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ for (step = 0; step < rbio->sector_nsteps; step++) {
++++++++++++ ++ ++ ++++++++ phys_addr_t paddr = rbio_stripe_paddr(rbio, stripe_nr,
++++++++++++ ++ ++ ++++++++ sectornr, step);
++++++++++++ ++ ++ ++++++++ void *kaddr = kmap_local_paddr(paddr);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ memset(kaddr, 0, min_t(u32, sectorsize, PAGE_SIZE));
++++++++++++ ++ ++ ++++++++ kunmap_local(kaddr);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ set_bit(i, rbio->stripe_uptodate_bitmap);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ set_bit(RBIO_PADDED_BIT, &rbio->flags);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * The whole stripe goes out: every vertical position is written.
++++++++++++ ++ ++ ++++++++ * Under bio_list_lock: until RBIO_RMW_LOCKED_BIT is set, merge_rbio()
++++++++++++ ++ ++ ++++++++ * can still OR another rbio's bits into this bitmap.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ spin_lock(&rbio->bio_list_lock);
++++++++++++ ++ ++ ++++++++ bitmap_set(&rbio->dbitmap, 0, rbio->stripe_nsectors);
++++++++++++ ++ ++ ++++++++ spin_unlock(&rbio->bio_list_lock);
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.padded);
++++++++++++ ++ ++ ++++++++ if (nr_unwritten)
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.padded_unwritten);
++++++++++++ ++ ++ ++++++++ return true;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++ + ++ ++ ++++++++/*
++++++++++ + ++ ++ ++++++++ * To determine if we need to read any sector from the disk.
++++++++++ + ++ ++ ++++++++ * Should only be utilized in RMW path, to skip cached rbio.
++++++++++ + ++ ++ ++++++++ */
++++++++++ + ++ ++ ++++++++static bool need_read_stripe_sectors(struct btrfs_raid_bio *rbio)
++++++++++ + ++ ++ ++++++++{
++++++++++ + ++ ++ ++++++++ int i;
++++++++++ + ++ ++ ++++++++
++++++++++ + ++ ++ ++++++++ for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) {
++++++++ + + ++ ++ ++++++++ phys_addr_t paddr = rbio->stripe_paddrs[i * rbio->sector_nsteps];
++++++++ + + ++ ++ ++++++++
++++++++ + + ++ ++ ++++++++ /*
++++++++ + + ++ ++ ++++++++ * We have a sector which doesn't have page nor uptodate,
++++++++ + + ++ ++ ++++++++ * thus this rbio can not be cached one, as cached one must
++++++++ + + ++ ++ ++++++++ * have all its data sectors present and uptodate.
++++++++ + + ++ ++ ++++++++ */
++++++++ + + ++ ++ ++++++++ if (paddr == INVALID_PADDR ||
++++++++ + + ++ ++ ++++++++ !test_bit(i, rbio->stripe_uptodate_bitmap))
++++++++ + + ++ ++ ++++++++ return true;
++++++++ + + ++ ++ ++++++++ }
++++++++ + + ++ ++ ++++++++ return false;
++++++++ + + ++ ++ ++++++++}
++++++++ + + ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Read the header of the tree block at data-stripe sector @i, if there is
++++++++++++ ++ ++ ++++++++ * one, from whichever copy of the sector this rbio is holding.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * @from_bio picks the source: the caller's pages (what this write is putting
++++++++++++ ++ ++ ++++++++ * there) or the stripe pages (what an RMW read found on the disk). Returns
++++++++++++ ++ ++ ++++++++ * false when the sector holds no tree block that belongs at this address --
++++++++++++ ++ ++ ++++++++ * free space, or the remains of one from a previous life of the stripe.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static bool rmw_peek_eb_generation(struct btrfs_raid_bio *rbio, u32 i,
++++++++++++ ++ ++ ++++++++ bool from_bio, u64 *generation)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
++++++++++++ ++ ++ ++++++++ const int stripe_nr = i / rbio->stripe_nsectors;
++++++++++++ ++ ++ ++++++++ const int sectornr = i % rbio->stripe_nsectors;
++++++++++++ ++ ++ ++++++++ const u64 logical = rbio->bioc->full_stripe_logical +
++++++++++++ ++ ++ ++++++++ (u64)stripe_nr * BTRFS_STRIPE_LEN +
++++++++++++ ++ ++ ++++++++ (u64)sectornr * fs_info->sectorsize;
++++++++++++ ++ ++ ++++++++ const phys_addr_t *paddrs = from_bio ? rbio->bio_paddrs
++++++++++++ ++ ++ ++++++++ : rbio->stripe_paddrs;
++++++++++++ ++ ++ ++++++++ struct btrfs_header *hdr;
++++++++++++ ++ ++ ++++++++ phys_addr_t paddr;
++++++++++++ ++ ++ ++++++++ u64 found_bytenr;
++++++++++++ ++ ++ ++++++++ bool bad_fsid;
++++++++++++ ++ ++ ++++++++ void *kaddr;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ paddr = paddrs[i * rbio->sector_nsteps];
++++++++++++ ++ ++ ++++++++ if (paddr == INVALID_PADDR)
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++ if (!from_bio && !test_bit(i, rbio->stripe_uptodate_bitmap))
++++++++++++ ++ ++ ++++++++ return false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ kaddr = kmap_local_paddr(paddr);
++++++++++++ ++ ++ ++++++++ hdr = kaddr;
++++++++++++ ++ ++ ++++++++ found_bytenr = btrfs_stack_header_bytenr(hdr);
++++++++++++ ++ ++ ++++++++ *generation = btrfs_stack_header_generation(hdr);
++++++++++++ ++ ++ ++++++++ bad_fsid = memcmp(hdr->fsid, fs_info->fs_devices->metadata_uuid,
++++++++++++ ++ ++ ++++++++ BTRFS_FSID_SIZE) != 0;
++++++++++++ ++ ++ ++++++++ kunmap_local(kaddr);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ return found_bytenr == logical && !bad_fsid;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Is @bytenr a tree block the committed extent tree still references? Read
++++++++++++ ++ ++ ++++++++ * the commit root without locking, as the caching thread does, so this is
++++++++++++ ++ ++ ++++++++ * safe from the raid56 worker with no transaction to join.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Returns 1 if it is (or the lookup failed), 0 if not, and -EAGAIN if the
++++++++++++ ++ ++ ++++++++ * answer needs a tree block that is not in memory. The caller holds the
++++++++++++ ++ ++ ++++++++ * full stripe lock, so it must not read from disk: a read of a block in
++++++++++++ ++ ++ ++++++++ * this stripe that needs reconstruction would wait for that lock.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static int rmw_block_referenced(struct btrfs_fs_info *fs_info, u64 bytenr,
++++++++++++ ++ ++ ++++++++ u64 header_gen, u64 *refs, u64 *extent_gen)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_root *extent_root = btrfs_extent_root(fs_info, bytenr);
++++++++++++ ++ ++ ++++++++ struct btrfs_path *path;
++++++++++++ ++ ++ ++++++++ struct btrfs_key key;
++++++++++++ ++ ++ ++++++++ int live = 1;
++++++++++++ ++ ++ ++++++++ int ret;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ *refs = 0;
++++++++++++ ++ ++ ++++++++ *extent_gen = 0;
++++++++++++ ++ ++ ++++++++ path = btrfs_alloc_path();
++++++++++++ ++ ++ ++++++++ if (!path)
++++++++++++ ++ ++ ++++++++ return 1;
++++++++++++ ++ ++ ++++++++ path->search_commit_root = 1;
++++++++++++ ++ ++ ++++++++ path->skip_locking = 1;
++++++++++++ ++ ++ ++++++++ path->nowait = 1;
++++++++++++ ++ ++ ++++++++ key.objectid = bytenr;
++++++++++++ ++ ++ ++++++++ key.type = BTRFS_METADATA_ITEM_KEY;
++++++++++++ ++ ++ ++++++++ key.offset = (u64)-1;
++++++++++++ ++ ++ ++++++++ if (!down_read_trylock(&fs_info->commit_root_sem)) {
++++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
++++++++++++ ++ ++ ++++++++ return -EAGAIN;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
++++++++++++ ++ ++ ++++++++ if (ret > 0)
++++++++++++ ++ ++ ++++++++ ret = btrfs_previous_extent_item(extent_root, path, bytenr);
++++++++++++ ++ ++ ++++++++ if (ret == 0) {
++++++++++++ ++ ++ ++++++++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
++++++++++++ ++ ++ ++++++++ if (key.objectid == bytenr &&
++++++++++++ ++ ++ ++++++++ (key.type == BTRFS_METADATA_ITEM_KEY ||
++++++++++++ ++ ++ ++++++++ key.type == BTRFS_EXTENT_ITEM_KEY)) {
++++++++++++ ++ ++ ++++++++ struct btrfs_extent_item *ei;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
++++++++++++ ++ ++ ++++++++ struct btrfs_extent_item);
++++++++++++ ++ ++ ++++++++ *refs = btrfs_extent_refs(path->nodes[0], ei);
++++++++++++ ++ ++ ++++++++ *extent_gen = btrfs_extent_generation(path->nodes[0], ei);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * An extent item at this bytenr from a LATER
++++++++++++ ++ ++ ++++++++ * transaction than the header on disk belongs to a
++++++++++++ ++ ++ ++++++++ * new block allocated here whose write has not landed
++++++++++++ ++ ++ ++++++++ * yet; the header is the dead previous occupant.
++++++++++++ ++ ++ ++++++++ * Only an item of the header's own generation makes
++++++++++++ ++ ++ ++++++++ * the block on disk live.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ live = (*refs != 0 && *extent_gen == header_gen);
++++++++++++ ++ ++ ++++++++ } else {
++++++++++++ ++ ++ ++++++++ live = 0;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ } else if (ret > 0) {
++++++++++++ ++ ++ ++++++++ live = 0;
++++++++++++ ++ ++ ++++++++ } else if (ret == -EAGAIN) {
++++++++++++ ++ ++ ++++++++ live = -EAGAIN;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ up_read(&fs_info->commit_root_sem);
++++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
++++++++++++ ++ ++ ++++++++ return live;
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Decide whether an uncovered metadata read-modify-write is a write hole or
++++++++++++ ++ ++ ++++++++ * only a cost, from the tree blocks it read: their parity is about to be
++++++++++++ ++ ++ ++++++++ * rewritten while their own sectors stay put.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * - Same generation as the blocks this write carries (meta_rmw_cur): the
++++++++++++ ++ ++ ++++++++ * block belongs to the running transaction, which a crash discards --
++++++++++++ ++ ++ ++++++++ * except a log tree block, which is durable once its fsync returns.
++++++++++++ ++ ++ ++++++++ * The audit does not tell log blocks apart, so it cannot see a write
++++++++++++ ++ ++ ++++++++ * hole over fsynced log blocks; keeping them out of shared stripes is
++++++++++++ ++ ++ ++++++++ * stripe_log_run's job.
++++++++++++ ++ ++ ++++++++ * - Older, and still referenced by the committed extent tree: a write hole
++++++++++++ ++ ++ ++++++++ * over committed metadata (meta_rmw_old).
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Comparing against this rbio's own blocks, not fs_info's committed
++++++++++++ ++ ++ ++++++++ * generation, keeps a concurrent commit from skewing the verdict.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static void audit_uncovered_rmw(struct btrfs_raid_bio *rbio)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
++++++++++++ ++ ++ ++++++++ const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits;
++++++++++++ ++ ++ ++++++++ const u32 nsectors = rbio->nr_data * rbio->stripe_nsectors;
++++++++++++ ++ ++ ++++++++ u64 write_gen = 0;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Tree blocks only: metadata and system groups. A mixed group's
++++++++++++ ++ ++ ++++++++ * sectors need not hold tree blocks.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ if (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ if (!(rbio->bioc->map_type & (BTRFS_BLOCK_GROUP_METADATA |
++++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Both loops read bio_paddrs to tell what this write covers, and the
++++++++++++ ++ ++ ++++++++ * paths that reach here do not all populate it (a cached rbio that
++++++++++++ ++ ++ ++++++++ * failed to pad never indexes). Idempotent, so just do it.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ index_rbio_pages(rbio);
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* The generation this write is putting into the stripe. */
++++++++++++ ++ ++ ++++++++ for (u32 i = 0; i < nsectors; i += sectors_per_tree) {
++++++++++++ ++ ++ ++++++++ u64 gen;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (rmw_peek_eb_generation(rbio, i, true, &gen))
++++++++++++ ++ ++ ++++++++ write_gen = max(write_gen, gen);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (!write_gen)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ for (u32 i = 0; i < nsectors; i += sectors_per_tree) {
++++++++++++ ++ ++ ++++++++ u64 gen;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Covered by this write: it is what we are putting there. */
++++++++++++ ++ ++ ++++++++ if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ if (!rmw_peek_eb_generation(rbio, i, false, &gen)) {
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_free);
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (gen >= write_gen) {
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_cur);
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * An older header is a write hole only if the committed extent
++++++++++++ ++ ++ ++++++++ * tree still references the block: a reclaimed stripe keeps the
++++++++++++ ++ ++ ++++++++ * headers of dead blocks, and tearing those harms nothing a
++++++++++++ ++ ++ ++++++++ * reader can reach. The commit root is what a crash would see.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ {
++++++++++++ ++ ++ ++++++++ const int stripe_nr = i / rbio->stripe_nsectors;
++++++++++++ ++ ++ ++++++++ const int sectornr = i % rbio->stripe_nsectors;
++++++++++++ ++ ++ ++++++++ const u64 logical = rbio->bioc->full_stripe_logical +
++++++++++++ ++ ++ ++++++++ (u64)stripe_nr * BTRFS_STRIPE_LEN +
++++++++++++ ++ ++ ++++++++ (u64)sectornr * fs_info->sectorsize;
++++++++++++ ++ ++ ++++++++ u64 refs;
++++++++++++ ++ ++ ++++++++ u64 egen;
++++++++++++ ++ ++ ++++++++ int ret;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ ret = rmw_block_referenced(fs_info, logical, gen, &refs,
++++++++++++ ++ ++ ++++++++ &egen);
++++++++++++ ++ ++ ++++++++ if (ret == -EAGAIN) {
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_unknown);
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (!ret) {
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_ghost);
++++++++++++ ++ ++ ++++++++ continue;
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_old);
++++++++++++ ++ ++ ++++++++ btrfs_warn_rl(fs_info,
++++++++++++ ++ ++ ++++++++"read-modify-write of stripe %llu rewrites parity over committed tree block %llu at generation %llu (refs %llu, extent generation %llu) while writing generation %llu: write hole",
++++++++++++ ++ ++ ++++++++ rbio->bioc->full_stripe_logical, logical,
++++++++++++ ++ ++ ++++++++ gen, refs, egen, write_gen);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++/*
++++++++++++ ++ ++ ++++++++ * Count and audit a metadata or system read-modify-write
++++++++++++ ++ ++ ++++++++ * (audit_uncovered_rmw()), and warn if it lands in a raid56 group that
++++++++++++ ++ ++ ++++++++ * stripe_meta does not cover. Silent when stripe_alloc is off.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++static void report_uncovered_rmw(struct btrfs_raid_bio *rbio)
++++++++++++ ++ ++ ++++++++{
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
++++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
++++++++++++ ++ ++ ++++++++ bool covered = false;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ /* Data groups are btrfs_is_stripe_alloc_bg()'s. */
++++++++++++ ++ ++ ++++++++ if ((rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA) &&
++++++++++++ ++ ++ ++++++++ !(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_METADATA))
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw);
++++++++++++ ++ ++ ++++++++ audit_uncovered_rmw(rbio);
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Every raid56 metadata and system group is a stripe_meta group while
++++++++++++ ++ ++ ++++++++ * stripe_alloc is on, and stripe_alloc refuses to enable on mixed
++++++++++++ ++ ++ ++++++++ * groups (btrfs_stripe_alloc_check_support()), so this warning is a
++++++++++++ ++ ++ ++++++++ * defensive check: it fires only if a tree block group that should be
++++++++++++ ++ ++ ++++++++ * covered is not.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, rbio->bioc->full_stripe_logical);
++++++++++++ ++ ++ ++++++++ if (bg) {
++++++++++++ ++ ++ ++++++++ covered = btrfs_is_stripe_meta_bg(bg);
++++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
++++++++++++ ++ ++ ++++++++ }
++++++++++++ ++ ++ ++++++++ if (covered)
++++++++++++ ++ ++ ++++++++ return;
++++++++++++ ++ ++ ++++++++ btrfs_warn_rl(fs_info,
++++++++++++ ++ ++ ++++++++"read-modify-write of full stripe %llu: this block group is not covered by stripe_alloc, the raid56 write hole applies to it",
++++++++++++ ++ ++ ++++++++ rbio->bioc->full_stripe_logical);
++++++++++++ ++ ++ ++++++++}
++++++++++++ ++ ++ ++++++++
++++++++ ++++++++++++++++++/*
++++++++ ++++++++++++++++++ * Return true if the rebuild of data sector @sector_nr of @stripe_nr can be
++++++++ ++++++++++++++++++ * trusted enough to write it back. It can when the sector has a csum: the
++++++++ ++++++++++++++++++ * rebuild was verified. Without a csum, it can only when another data sector
++++++++ ++++++++++++++++++ * of the same vertical stripe also failed and has a csum: the two were
++++++++ ++++++++++++++++++ * rebuilt together from P and Q (on RAID6 that is also how a stale sector
++++++++ ++++++++++++++++++ * without a csum is found, see recover_vertical_retry()), and the csum that
++++++++ ++++++++++++++++++ * verified one verifies the other. A lone rebuild without a csum came from
++++++++ ++++++++++++++++++ * parity that nothing checked, and the copy on the disk may still be right.
++++++++ ++++++++++++++++++ */
++++++++ ++++++++++++++++++static bool rmw_repair_verified(struct btrfs_raid_bio *rbio, int stripe_nr,
++++++++ ++++++++++++++++++ int sector_nr)
++++++++ ++++++++++++++++++{
++++++++ ++++++++++++++++++ if (!rbio->csum_bitmap)
++++++++ ++++++++++++++++++ return false;
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++ for (int i = 0; i < rbio->nr_data; i++) {
++++++++ ++++++++++++++++++ const int index = rbio_sector_index(rbio, i, sector_nr);
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++ if (!test_bit(index, rbio->csum_bitmap))
++++++++ ++++++++++++++++++ continue;
++++++++ ++++++++++++++++++ if (i == stripe_nr || test_bit(index, rbio->error_bitmap))
++++++++ ++++++++++++++++++ return true;
++++++++ ++++++++++++++++++ }
++++++++ ++++++++++++++++++ return false;
++++++++ ++++++++++++++++++}
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++/*
++++++++ ++++++++++++++++++ * After the read phase of a sub-stripe write, return the sectors it repaired
++++++++ ++++++++++++++++++ * that the write phase should also write back, or NULL if there are none.
++++++++ ++++++++++++++++++ *
++++++++ ++++++++++++++++++ * recover_sectors() leaves the error bit set on every sector it rebuilt: a
++++++++ ++++++++++++++++++ * data sector that failed its csum or its read, every sector of a missing
++++++++ ++++++++++++++++++ * device, and the extra stripe that recover_vertical_retry() found stale.
++++++++ ++++++++++++++++++ * The rebuilt sectors are in the stripe pages and the new P and Q are
++++++++ ++++++++++++++++++ * generated from them, so without a write-back the disk copies stay stale
++++++++ ++++++++++++++++++ * until read repair or scrub finds them.
++++++++ ++++++++++++++++++ *
++++++++ ++++++++++++++++++ * Nothing is written to a missing device. A data sector is written back only
++++++++ ++++++++++++++++++ * if its rebuild is verified (rmw_repair_verified()). P and Q are generated
++++++++ ++++++++++++++++++ * from the data either way, so a failed or stale one is always written back.
++++++++ ++++++++++++++++++ *
++++++++ ++++++++++++++++++ * This rbio has not written anything yet, so none of these is a retry of a
++++++++ ++++++++++++++++++ * write that just failed here.
++++++++ ++++++++++++++++++ */
++++++++ ++++++++++++++++++static unsigned long *rmw_repaired_sectors(struct btrfs_raid_bio *rbio)
++++++++ ++++++++++++++++++{
++++++++ ++++++++++++++++++ unsigned long *repaired = NULL;
++++++++ ++++++++++++++++++ int total_sector_nr;
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++ for_each_set_bit(total_sector_nr, rbio->error_bitmap, rbio->nr_sectors) {
++++++++ ++++++++++++++++++ const int stripe = total_sector_nr / rbio->stripe_nsectors;
++++++++ ++++++++++++++++++ const int sectornr = total_sector_nr % rbio->stripe_nsectors;
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++ if (!rbio->bioc->stripes[stripe].dev->bdev)
++++++++ ++++++++++++++++++ continue;
++++++++ ++++++++++++++++++ if (stripe < rbio->nr_data &&
++++++++ ++++++++++++++++++ !rmw_repair_verified(rbio, stripe, sectornr))
++++++++ ++++++++++++++++++ continue;
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++ if (!repaired) {
++++++++ ++++++++++++++++++ /* Without memory the sectors just stay stale on disk. */
++++++++ ++++++++++++++++++ repaired = bitmap_zalloc(rbio->nr_sectors, GFP_NOFS);
++++++++ ++++++++++++++++++ if (!repaired)
++++++++ ++++++++++++++++++ return NULL;
++++++++ ++++++++++++++++++ }
++++++++ ++++++++++++++++++ set_bit(total_sector_nr, repaired);
++++++++ ++++++++++++++++++ }
++++++++ ++++++++++++++++++ return repaired;
++++++++ ++++++++++++++++++}
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++/*
++++++++ ++++++++++++++++++ * Add write bios for the repaired sectors that the write phase does not write
++++++++ ++++++++++++++++++ * anyway: data sectors no bio of this rbio covers, and P/Q outside the
++++++++ ++++++++++++++++++ * vertical stripes being written. The others are dropped from @repaired.
++++++++ ++++++++++++++++++ * The replace target gets the same sectors as its source.
++++++++ ++++++++++++++++++ */
++++++++ ++++++++++++++++++static int rmw_assemble_repair_bios(struct btrfs_raid_bio *rbio,
++++++++ ++++++++++++++++++ struct bio_list *bio_list,
++++++++ ++++++++++++++++++ unsigned long *repaired)
++++++++ ++++++++++++++++++{
++++++++ ++++++++++++++++++ int total_sector_nr;
++++++++ ++++++++++++++++++ int ret;
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++ for_each_set_bit(total_sector_nr, repaired, rbio->nr_sectors) {
++++++++ ++++++++++++++++++ const int stripe = total_sector_nr / rbio->stripe_nsectors;
++++++++ ++++++++++++++++++ const int sectornr = total_sector_nr % rbio->stripe_nsectors;
++++++++ ++++++++++++++++++ phys_addr_t *paddrs;
++++++++ ++++++++++++++++++ bool written;
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++ if (stripe < rbio->nr_data)
++++++++ ++++++++++++++++++ written = sector_paddrs_in_rbio(rbio, stripe, sectornr, 1);
++++++++ ++++++++++++++++++ else
++++++++ ++++++++++++++++++ written = test_bit(sectornr, &rbio->dbitmap);
++++++++ ++++++++++++++++++ if (written) {
++++++++ ++++++++++++++++++ clear_bit(total_sector_nr, repaired);
++++++++ ++++++++++++++++++ continue;
++++++++ ++++++++++++++++++ }
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++ paddrs = rbio_stripe_paddrs(rbio, stripe, sectornr);
++++++++ ++++++++++++++++++ ret = rbio_add_io_paddrs(rbio, bio_list, paddrs, stripe,
++++++++ ++++++++++++++++++ sectornr, REQ_OP_WRITE);
++++++++ ++++++++++++++++++ if (ret)
++++++++ ++++++++++++++++++ goto error;
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++ if (likely(!rbio->bioc->replace_nr_stripes) ||
++++++++ ++++++++++++++++++ stripe != rbio->bioc->replace_stripe_src)
++++++++ ++++++++++++++++++ continue;
++++++++ ++++++++++++++++++ ret = rbio_add_io_paddrs(rbio, bio_list, paddrs,
++++++++ ++++++++++++++++++ rbio->real_stripes, sectornr,
++++++++ ++++++++++++++++++ REQ_OP_WRITE);
++++++++ ++++++++++++++++++ if (ret)
++++++++ ++++++++++++++++++ goto error;
++++++++ ++++++++++++++++++ }
++++++++ ++++++++++++++++++ return 0;
++++++++ ++++++++++++++++++error:
++++++++ ++++++++++++++++++ bio_list_put(bio_list);
++++++++ ++++++++++++++++++ return -EIO;
++++++++ ++++++++++++++++++}
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++/*
++++++++ ++++++++++++++++++ * A write-back that fails leaves the sector as stale as it was before this
++++++++ ++++++++++++++++++ * write, so it does not count against the tolerance of the write itself.
++++++++ ++++++++++++++++++ */
++++++++ ++++++++++++++++++static void rmw_finish_repair_bios(struct btrfs_raid_bio *rbio,
++++++++ ++++++++++++++++++ unsigned long *repaired)
++++++++ ++++++++++++++++++{
++++++++ ++++++++++++++++++ int written = 0;
++++++++ ++++++++++++++++++ int failed = 0;
++++++++ ++++++++++++++++++ int total_sector_nr;
++++++++ ++++++++++++++++++
++++++++ ++++++++++++++++++ for_each_set_bit(total_sector_nr, repaired, rbio->nr_sectors) {
++++++++ ++++++++++++++++++ written++;
++++++++ ++++++++++++++++++ if (test_and_clear_bit(total_sector_nr, rbio->error_bitmap))
++++++++ ++++++++++++++++++ failed++;
++++++++ ++++++++++++++++++ }
++++++++ ++++++++++++++++++ if (written)
++++++++ ++++++++++++++++++ btrfs_debug_rl(rbio->bioc->fs_info,
++++++++ ++++++++++++++++++ "sub-stripe write for full stripe %llu wrote back %d repaired sectors, %d failed",
++++++++ ++++++++++++++++++ rbio->bioc->full_stripe_logical, written, failed);
++++++++ ++++++++++++++++++}
++++++++ ++++++++++++++++++
++++++++ + + ++ ++ ++++++++static void rmw_rbio(struct btrfs_raid_bio *rbio)
++++++++ + + ++ ++ ++++++++{
++++++++ + + ++ ++ ++++++++ struct bio_list bio_list;
++++++++ + + ++ ++ ++++++++ int sectornr;
++++++++ ++++++++++++++++++ unsigned long *repaired = NULL;
++++++++ + + ++ ++ ++++++++ int ret = 0;
++++++++ + + ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Write-hole invariant check: every write must land in a stripe
++++++++++++ ++ ++ ++++++++ * covered by a live stripe run when stripe-exclusive allocation is
++++++++++++ ++ ++ ++++++++ * enabled. The bios gathered in this rbio have not reported their
++++++++++++ ++ ++ ++++++++ * IO done yet, so their runs cannot drain under us.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ btrfs_stripe_check_write(rbio->bioc->fs_info,
++++++++++++ ++ ++ ++++++++ rbio->bioc->full_stripe_logical,
++++++++++++ ++ ++ ++++++++ !rbio_is_full(rbio));
++++++++++++ ++ ++ ++++++++
++++++++ + + ++ ++ ++++++++ /*
++++++++ + + ++ ++ ++++++++ * Allocate the pages for parity first, as P/Q pages will always be
++++++++ + + ++ ++ ++++++++ * needed for both full-stripe and sub-stripe writes.
++++++++ + + ++ ++ ++++++++ */
++++++++ + + ++ ++ ++++++++ ret = alloc_rbio_parity_pages(rbio);
++++++++ + + ++ ++ ++++++++ if (ret < 0)
++++++++ + + ++ ++ ++++++++ goto out;
++++++++ + + ++ ++ ++++++++
++++++++ + + ++ ++ ++++++++ /*
++++++++ + + ++ ++ ++++++++ * Either full stripe write, or we have every data sector already
++++++++ + + ++ ++ ++++++++ * cached, can go to write path immediately.
++++++++++++ ++ ++ ++++++++ *
++++++++++++ ++ ++ ++++++++ * Try padding before trusting the cache: a cached rbio can hold pages
++++++++++++ ++ ++ ++++++++ * from the stripe's previous life, before its extents were freed and
++++++++++++ ++ ++ ++++++++ * the stripe re-claimed, and parity computed from them matches the
++++++++++++ ++ ++ ++++++++ * disk only if every uncovered column still holds that data. Padding
++++++++++++ ++ ++ ++++++++ * writes every uncovered column, so the stripe needs nothing on disk.
++++++++ + + ++ ++ ++++++++ */
- - if (!rbio_is_full(rbio) && need_read_stripe_sectors(rbio)) {
++++++++++++ ++ ++ ++++++++ if (!rbio_is_full(rbio) && !rmw_try_pad_full(rbio) &&
++++++++++++ ++ ++ ++++++++ need_read_stripe_sectors(rbio)) {
++++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
++++++++++++ ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /*
++++++++++++ ++ ++ ++++++++ * Every data read-modify-write, parked or not. With
++++++++++++ ++ ++ ++++++++ * allow_rmw empty and a covered block group this is the
++++++++++++ ++ ++ ++++++++ * count that should be zero.
++++++++++++ ++ ++ ++++++++ */
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw);
++++++++++++ ++ ++ ++++++++ if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA))
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_reads);
++++++++++++ ++ ++ ++++++++ rbio_count_rmw_reason(rbio);
++++++++++++ ++ ++ ++++++++ /* A parked-then-flushed write that still had to read. */
++++++++++++ ++ ++ ++++++++ if (test_bit(RBIO_WAS_PARKED_BIT, &rbio->flags))
++++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.rmw_reads);
++++++++ + + ++ ++ ++++++++ /*
++++++++ + + ++ ++ ++++++++ * Now we're doing sub-stripe write, also need all data stripes
++++++++ + + ++ ++ ++++++++ * to do the full RMW.
++++++++ + + ++ ++ ++++++++ */
++++++++ + + ++ ++ ++++++++ ret = alloc_rbio_data_pages(rbio);
++++++++ + + ++ ++ ++++++++ if (ret < 0)
++++++++ + + ++ ++ ++++++++ goto out;
++++++++ + + ++ ++ ++++++++
++++++++ + + ++ ++ ++++++++ index_rbio_pages(rbio);
++++++++ + + ++ ++ ++++++++
++++++++ + + ++ ++ ++++++++ ret = rmw_read_wait_recover(rbio);
++++++++ + + ++ ++ ++++++++ if (ret < 0)
++++++++ + + ++ ++ ++++++++ goto out;
++++++++ ++++++++++++++++++ repaired = rmw_repaired_sectors(rbio);
++++++++ + + ++ ++ ++++++++ }
++++++++ + + ++ ++ ++++++++
++++++++++++ ++ ++ ++++++++ /* Still a sub-stripe write: this stripe is being modified in place. */
++++++++++++ ++ ++ ++++++++ if (!rbio_is_full(rbio) && !test_bit(RBIO_PADDED_BIT, &rbio->flags))
++++++++++++ ++ ++ ++++++++ report_uncovered_rmw(rbio);
++++++++++++ ++ ++ ++++++++
++++++++ + + ++ ++ ++++++++ /*
++++++++ + + ++ ++ ++++++++ * At this stage we're not allowed to add any new bios to the
++++++++ + + ++ ++ ++++++++ * bio list any more, anyone else that wants to change this stripe
++++++++ + + ++ ++ ++++++++ * needs to do their own rmw.
++++++++ + + ++ ++ ++++++++ */
++++++++ + + ++ ++ ++++++++ spin_lock(&rbio->bio_list_lock);
++++++++ + + ++ ++ ++++++++ set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
++++++++ + + ++ ++ ++++++++ spin_unlock(&rbio->bio_list_lock);
++++++++ + + ++ ++ ++++++++
++++++++ + + ++ ++ ++++++++ bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors);
++++++++ + + ++ ++ ++++++++
++++++++ + + ++ ++ ++++++++ index_rbio_pages(rbio);
/*
* We don't cache full rbios because we're assuming
q_paddr = INVALID_PADDR;
}
- - * nothing back to the scrubbed device. A running dev-replace still
- - * copies to its target.
++++++++++ ++++++++++++++++ /* A read-only scrub only checks: submit no write. */
++++++++++ ++++++++++++++++ if (rbio->scrub_readonly)
++++++++++ ++++++++++++++++ goto submit_write;
++++++++++ ++++++++++++++++
+++++++++ ++++++++++++++ + /*
+++++++++ ++++++++++++++ + * After a transaction abort or another filesystem error, write
++++++++++ ++++++++++++++++ * nothing back to the scrubbed devices, rebuilt data included. A
++++++++++ ++++++++++++++++ * running dev-replace still copies parity to its target.
+++++++++ ++++++++++++++ + */
+++++++++ ++++++++++++++ + if (!btrfs_can_submit_repair_write(rbio->bioc->fs_info))
+++++++++ ++++++++++++++ + goto replace;
+++++++++ ++++++++++++++ +
++++++++++ ++++++++++++++++ /* Write the data sectors rebuilt from P and Q. */
++++++++++ ++++++++++++++++ for_each_set_bit(sectornr, &rebuilt, rbio->stripe_nsectors) {
++++++++++ ++++++++++++++++ const int stripe = rebuilt_stripe[sectornr];
++++++++++ ++++++++++++++++ const struct btrfs_io_stripe *io_stripe = &bioc->stripes[stripe];
++++++++++ ++++++++++++++++
++++++++++ ++++++++++++++++ btrfs_warn_rl(bioc->fs_info,
++++++++++ ++++++++++++++++"scrub: raid6 data at logical %llu on devid %llu physical %llu did not match P and Q, rebuilt it",
++++++++++ ++++++++++++++++ bioc->full_stripe_logical +
++++++++++ ++++++++++++++++ btrfs_stripe_nr_to_offset(stripe) +
++++++++++ ++++++++++++++++ ((u64)sectornr << bioc->fs_info->sectorsize_bits),
++++++++++ ++++++++++++++++ io_stripe->dev->devid,
++++++++++ ++++++++++++++++ io_stripe->physical +
++++++++++ ++++++++++++++++ ((u64)sectornr << bioc->fs_info->sectorsize_bits));
++++++++++ ++++++++++++++++ ret = rbio_add_io_paddrs(rbio, &bio_list,
++++++++++ ++++++++++++++++ rbio_stripe_paddrs(rbio, stripe, sectornr),
++++++++++ ++++++++++++++++ stripe, sectornr, REQ_OP_WRITE);
++++++++++ ++++++++++++++++ if (ret)
++++++++++ ++++++++++++++++ goto cleanup;
++++++++++ ++++++++++++++++ }
++++++++++ ++++++++++++++++
/*
* time to start writing. Make bios for everything from the
* higher layers (the bio_list in our rbio) and our p/q. Ignore