wait_var_event(&bg->reservations, !atomic_read(&bg->reservations));
}
- - if (bg->start == READ_ONCE(fs_info->data_reloc_bg))
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Open stripe runs
+++++++++++++ ++ ++++++++ * ================
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * The raid56 stripe allocation policy only allocates from "open stripe
+++++++++++++ ++ ++++++++ * runs": contiguous, stripe-aligned regions claimed whole from the free
+++++++++++++ ++ ++++++++ * space cache (btrfs_claim_free_stripe_run()) and filled strictly
+++++++++++++ ++ ++++++++ * sequentially. A run is closed when it is exhausted, when it loses a
+++++++++++++ ++ ++++++++ * band collision (stripe_run_place()), when a private run is too small and
+++++++++++++ ++ ++++++++ * is replaced, when a metadata run belongs to an earlier transaction, or
+++++++++++++ ++ ++++++++ * when it is retired at transaction commit. A closed run is never
+++++++++++++ ++ ++++++++ * reopened, and its unallocated tail returns to the free space cache
+++++++++++++ ++ ++++++++ * where, being part of a partially filled stripe, it cannot be claimed
+++++++++++++ ++ ++++++++ * until the whole stripe frees. So every stripe is written within one
+++++++++++++ ++ ++++++++ * run's lifetime, which closes the raid56 write hole for these block
+++++++++++++ ++ ++++++++ * groups.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Each run counts the bytes reserved from it whose IO has not completed
+++++++++++++ ++ ++++++++ * (inflight_bytes). At commit, while the transaction still accepts joins
+++++++++++++ ++ ++++++++ * (before TRANS_STATE_COMMIT_DOING), btrfs_retire_open_stripes() closes
+++++++++++++ ++ ++++++++ * the data runs opened before the commit and waits for them to drain. The
+++++++++++++ ++ ++++++++ * wait must be a pure data-IO wait, never an ordered extent completion
+++++++++++++ ++ ++++++++ * wait, and must happen where joins still succeed: a task holding
+++++++++++++ ++ ++++++++ * unreported run bytes may block on a join (e.g. a chunk allocation during
+++++++++++++ ++ ++++++++ * delalloc). Metadata runs are retired after tree writeback
+++++++++++++ ++ ++++++++ * (btrfs_retire_meta_stripes()); persistent NOCOW runs survive commits.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Runs opened while the commit is in progress carry a newer open_seq and
+++++++++++++ ++ ++++++++ * belong to the next window. An ordered extent from such a run that
+++++++++++++ ++ ++++++++ * completes before the commit ends sees that its run's open_seq is not
+++++++++++++ ++ ++++++++ * retired by this transaction and defers its file extent to the next one.
+++++++++++++ ++ ++++++++ * The membership add, the sequence read and the run installation happen
+++++++++++++ ++ ++++++++ * atomically under fs_info->open_stripe_lock, so every run is either seen
+++++++++++++ ++ ++++++++ * by the walk that retires its window or stamped with a sequence that
+++++++++++++ ++ ++++++++ * forces the deferral.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Lock order: fs_info->open_stripe_lock outside bg->stripe_run_lock. The
+++++++++++++ ++ ++++++++ * stripe_run_lock is taken with interrupts disabled, which no current
+++++++++++++ ++ ++++++++ * caller needs: completed IO is reported from bio completion work, which
+++++++++++++ ++ ++++++++ * runs in task context. Membership in fs_info->open_stripe_bgs holds a
+++++++++++++ ++ ++++++++ * block group reference, dropped by the retire walk once the group has no
+++++++++++++ ++ ++++++++ * runs left, or by btrfs_free_open_stripe_runs() at teardown.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++static bool stripe_extents_owned_by(struct btrfs_fs_info *fs_info,
+++++++++++++ ++ ++++++++ u64 root_id, u64 ino, u64 start, u64 len,
+++++++++++++ ++ ++++++++ u64 *frontier, bool nowait);
+++++++++++++ ++ ++++++++static bool stripe_range_run_free(struct btrfs_block_group *bg, u64 start,
+++++++++++++ ++ ++++++++ u64 len);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++struct btrfs_open_stripe_run {
+++++++++++++ ++ ++++++++ struct list_head list; /* bg->open_stripe_runs */
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ u64 start;
+++++++++++++ ++ ++++++++ u64 end; /* exclusive; shrunk to offset on close */
+++++++++++++ ++ ++++++++ u64 offset; /* next unallocated byte */
+++++++++++++ ++ ++++++++ u64 inflight_bytes; /* reserved bytes with data IO pending */
+++++++++++++ ++ ++++++++ u64 open_seq; /* fs_info->stripe_retire_seq at open */
+++++++++++++ ++ ++++++++ u64 transid; /* fs_info->generation at open */
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Private runs are found by (root, owner): root is the subvolume's
+++++++++++++ ++ ++++++++ * root id, owner the inode number (LOG, NOCOW), a shared LOG key
+++++++++++++ ++ ++++++++ * (bit 63 set, see stripe_run_owner()), or for META_LOG the root id
+++++++++++++ ++ ++++++++ * again. Inode numbers are only unique within a subvolume.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ u64 root;
+++++++++++++ ++ ++++++++ u64 owner;
+++++++++++++ ++ ++++++++ enum btrfs_stripe_run_class class;
+++++++++++++ ++ ++++++++ bool open; /* accepting allocations */
+++++++++++++ ++ ++++++++ bool persistent; /* survives commits; see close_block_group_stripe_runs() */
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Relocation's run: kept after it closes and drains, until relocation
+++++++++++++ ++ ++++++++ * releases it (btrfs_release_stripe_reloc_runs()). See
+++++++++++++ ++ ++++++++ * run_freed_on_drain().
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ bool retained;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * One bit per sector of the run, set while the sector holds an
+++++++++++++ ++ ++++++++ * allocation someone still intends to write. Padding uses it to find
+++++++++++++ ++ ++++++++ * dead sectors below the frontier (allocations abandoned before they
+++++++++++++ ++ ++++++++ * were written), which are as safe to zero-fill as space above it.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ unsigned long *live;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Sectors of @live whose data has been written (raid56 write
+++++++++++++ ++ ++++++++ * completions, btrfs_stripe_run_mark_written()); same size as @live.
+++++++++++++ ++ ++++++++ * A stripe with live sectors not yet written is still waiting for
+++++++++++++ ++ ++++++++ * writes, and its cached rbio stays pinned (raid56.c, cache_rbio()).
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ unsigned long *written;
+++++++++++++ ++ ++++++++ u32 live_nbits; /* sectors @live covers; caps growth */
+++++++++++++ ++ ++++++++ /* META_LOG: the owning subvolume's log_transid this run belongs to. */
+++++++++++++ ++ ++++++++ u64 log_transid;
+++++++++++++ ++ ++++++++};
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++static void btrfs_dump_open_stripe_runs(struct btrfs_block_group *bg)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list)
+++++++++++++ ++ ++++++++ btrfs_warn(bg->fs_info,
+++++++++++++ ++ ++++++++" run %llu..%llu class %d frontier %llu open %d inflight %llu transid %llu",
+++++++++++++ ++ ++++++++ run->start, run->end, run->class, run->offset,
+++++++++++++ ++ ++++++++ run->open, run->inflight_bytes, run->transid);
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++static bool bg_has_undrained_meta_runs(struct btrfs_block_group *bg,
+++++++++++++ ++ ++++++++ u64 transid)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ bool ret = false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (!btrfs_stripe_run_class_is_meta(run->class))
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * An open run here belongs to the running transaction: the
+++++++++++++ ++ ++++++++ * retirer closed every run it is responsible for before its
+++++++++++++ ++ ++++++++ * wait, so this one was opened while it waited, and drains
+++++++++++++ ++ ++++++++ * when that transaction's own commit writes its tree blocks.
+++++++++++++ ++ ++++++++ * Waiting on it would be waiting for writes this commit never
+++++++++++++ ++ ++++++++ * issued.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (run->open)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Likewise a run the running transaction opened and a band
+++++++++++++ ++ ++++++++ * collision or eviction closed early: its blocks are written,
+++++++++++++ ++ ++++++++ * and its stripes settled, by that transaction's own commit.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (run->transid > transid)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (run->inflight_bytes) {
+++++++++++++ ++ ++++++++ ret = true;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++static bool open_stripe_run_drained(const struct btrfs_open_stripe_run *run)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ return !run->open && run->inflight_bytes == 0;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Round to a full stripe.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * full_stripe_len is nr_data_stripes * stripe_len, which is not a power of
+++++++++++++ ++ ++++++++ * two for most raid56 geometries (3 data stripes: 192K), so the mask-based
+++++++++++++ ++ ++++++++ * round_down(), round_up() and IS_ALIGNED() must not be used. Divide
+++++++++++++ ++ ++++++++ * instead.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static u64 stripe_round_down(u64 val, u64 fsl)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ return div64_u64(val, fsl) * fsl;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++static u64 stripe_round_up(u64 val, u64 fsl)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ return stripe_round_down(val + fsl - 1, fsl);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++static bool stripe_is_aligned(u64 val, u64 fsl)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ return stripe_round_down(val, fsl) == val;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Band for a run with @remaining bytes of unallocated space: the power-of-two
+++++++++++++ ++ ++++++++ * bucket of its remaining sectors (band k == remaining in [2^k, 2^(k+1))
+++++++++++++ ++ ++++++++ * sectors). Runs are kept at most one per band so an allocation can take the
+++++++++++++ ++ ++++++++ * smallest band that fits.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static int stripe_run_band(const struct btrfs_block_group *bg, u64 remaining)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ u32 sectors = remaining >> bg->fs_info->sectorsize_bits;
+++++++++++++ ++ ++++++++ int band;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!sectors)
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++ band = fls(sectors) - 1;
+++++++++++++ ++ ++++++++ if (band >= BTRFS_STRIPE_RUN_NR_BANDS)
+++++++++++++ ++ ++++++++ band = BTRFS_STRIPE_RUN_NR_BANDS - 1;
+++++++++++++ ++ ++++++++ return band;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Growth headroom for a run's liveness map, in full stripes.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * The map is sized once and never reallocated, because the only place a run
+++++++++++++ ++ ++++++++ * grows -- the growable frontier in btrfs_alloc_from_open_stripe() -- holds
+++++++++++++ ++ ++++++++ * bg->stripe_run_lock with interrupts disabled, where nothing may sleep.
+++++++++++++ ++ ++++++++ * Sizing it beyond the run's initial span is what lets that path work at
+++++++++++++ ++ ++++++++ * all; a run that would outgrow its map stops growing instead, and the
+++++++++++++ ++ ++++++++ * allocation opens a fresh run.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++#define BTRFS_STRIPE_RUN_LIVE_STRIPES 32
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Allocate a run's liveness map. Must be called before taking
+++++++++++++ ++ ++++++++ * bg->stripe_run_lock: this sleeps. Sets *nbits to what the map covers.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static unsigned long *alloc_run_live_map(struct btrfs_block_group *bg,
+++++++++++++ ++ ++++++++ u64 run_len, u32 *nbits)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ const u64 span = max_t(u64, run_len,
+++++++++++++ ++ ++++++++ (u64)BTRFS_STRIPE_RUN_LIVE_STRIPES *
+++++++++++++ ++ ++++++++ bg->full_stripe_len);
+++++++++++++ ++ ++++++++ unsigned long *map;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ *nbits = span >> bg->fs_info->sectorsize_bits;
+++++++++++++ ++ ++++++++ map = bitmap_zalloc(*nbits, GFP_NOFS);
+++++++++++++ ++ ++++++++ if (!map)
+++++++++++++ ++ ++++++++ *nbits = 0;
+++++++++++++ ++ ++++++++ return map;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/* Mark [start, start+len) live (allocated) or dead (abandoned). */
+++++++++++++ ++ ++++++++static void run_live_set(struct btrfs_open_stripe_run *run, u64 start, u64 len,
+++++++++++++ ++ ++++++++ bool live)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
+++++++++++++ ++ ++++++++ u32 first, nr;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!run->live || !run->live_nbits || start < run->start)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ first = (start - run->start) >> bits;
+++++++++++++ ++ ++++++++ nr = len >> bits;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Growth is capped at the map size, so this should be unreachable. If
+++++++++++++ ++ ++++++++ * it happens, retire the map: btrfs_stripe_run_pad_mask() then declines
+++++++++++++ ++ ++++++++ * the run, and the write falls back to read-modify-write instead of
+++++++++++++ ++ ++++++++ * zero-filling a sector that may be live.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (unlikely(first + nr > run->live_nbits)) {
+++++++++++++ ++ ++++++++ WARN_ON_ONCE(1);
+++++++++++++ ++ ++++++++ run->live_nbits = 0;
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (live)
+++++++++++++ ++ ++++++++ bitmap_set(run->live, first, nr);
+++++++++++++ ++ ++++++++ else
+++++++++++++ ++ ++++++++ bitmap_clear(run->live, first, nr);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * A run is dead when every byte it handed out has been freed again
+++++++++++++ ++ ++++++++ * (btrfs_stripe_run_note_free() clears the live bits). It then protects no
+++++++++++++ ++ ++++++++ * data, and the whole free stripes inside its range stay unclaimable
+++++++++++++ ++ ++++++++ * (btrfs_stripe_run_range_usable()) until it is closed.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Requires inflight_bytes == 0, since an in-flight write must keep its
+++++++++++++ ++ ++++++++ * stripes, and a valid live map (live_nbits == 0 means liveness is
+++++++++++++ ++ ++++++++ * unknown). Caller holds bg->stripe_run_lock.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static bool open_stripe_run_dead(const struct btrfs_open_stripe_run *run)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ if (run->inflight_bytes)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ if (run->offset <= run->start)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ if (!run->live || !run->live_nbits)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ return bitmap_empty(run->live, run->live_nbits);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Whether a run is freed as soon as it drains. Two kinds must outlive the
+++++++++++++ ++ ++++++++ * drain, because later writes to their stripes still need a covering run to
+++++++++++++ ++ ++++++++ * park against or pad from:
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * - Metadata runs: writeback may submit a stripe's tree blocks as separate
+++++++++++++ ++ ++++++++ * rbios, so the run drains after the first one. They are freed by
+++++++++++++ ++ ++++++++ * btrfs_retire_meta_stripes() once every tree block is on disk.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * - Relocation runs: relocation preallocates its destination extents (no
+++++++++++++ ++ ++++++++ * IO, so the run drains at once) and writes them only after the commit
+++++++++++++ ++ ++++++++ * that ends relocate_block_group(). They are retained until relocation
+++++++++++++ ++ ++++++++ * releases them (btrfs_release_stripe_reloc_runs()). Meanwhile their
+++++++++++++ ++ ++++++++ * stripes hold only relocation's own copies, and the closed run keeps its
+++++++++++++ ++ ++++++++ * range unclaimable.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static bool run_freed_on_drain(const struct btrfs_open_stripe_run *run)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ return !btrfs_stripe_run_class_is_meta(run->class) && !run->retained;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++static void free_open_stripe_run(struct btrfs_block_group *bg,
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ ASSERT(open_stripe_run_drained(run));
+++++++++++++ ++ ++++++++ list_del(&run->list);
+++++++++++++ ++ ++++++++ bitmap_free(run->live);
+++++++++++++ ++ ++++++++ bitmap_free(run->written);
+++++++++++++ ++ ++++++++ kfree(run);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Wholly free stripes inside this run's range were counted as trapped
+++++++++++++ ++ ++++++++ * by the last stripe_unusable scan (btrfs_stripe_run_stranded_stripes());
+++++++++++++ ++ ++++++++ * with the run gone they are claimable again, so have the next commit
+++++++++++++ ++ ++++++++ * rescan the group and credit them back.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (READ_ONCE(bg->stripe_unusable_ready)) {
+++++++++++++ ++ ++++++++ set_bit(BLOCK_GROUP_FLAG_STRIPE_UNUSABLE_DIRTY, &bg->runtime_flags);
+++++++++++++ ++ ++++++++ atomic64_inc(&bg->fs_info->stripe_run_stats.strand_rescan_marks);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * A run has drained. Free it if nothing needs it any more; otherwise
+++++++++++++ ++ ++++++++ * (metadata or retained relocation runs) wake the waiters directly, since a
+++++++++++++ ++ ++++++++ * report can arrive after btrfs_retire_meta_stripes() started waiting, e.g.
+++++++++++++ ++ ++++++++ * when an unwritten tree block is freed.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static void open_stripe_run_drain_reported(struct btrfs_block_group *bg,
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ if (!open_stripe_run_drained(run))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ if (run_freed_on_drain(run))
+++++++++++++ ++ ++++++++ free_open_stripe_run(bg, run);
+++++++++++++ ++ ++++++++ else
+++++++++++++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Close a run: no further allocation may join it, and its unallocated tail
+++++++++++++ ++ ++++++++ * must be returned to the free space cache by the caller (after dropping the
+++++++++++++ ++ ++++++++ * lock). The caller has already removed @run from bg->open_stripe[][].
+++++++++++++ ++ ++++++++ * Caller holds bg->stripe_run_lock. Returns the tail length, sets *tail_start.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static u64 close_open_stripe_run(struct btrfs_block_group *bg,
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run,
+++++++++++++ ++ ++++++++ u64 *tail_start)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ u64 tail_len;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ ASSERT(run->open);
+++++++++++++ ++ ++++++++ run->open = false;
+++++++++++++ ++ ++++++++ *tail_start = run->offset;
+++++++++++++ ++ ++++++++ tail_len = run->end - run->offset;
+++++++++++++ ++ ++++++++ run->end = run->offset;
+++++++++++++ ++ ++++++++ bg->stripe_open_remainder -= tail_len;
+++++++++++++ ++ ++++++++ if (open_stripe_run_drained(run) && !run->retained)
+++++++++++++ ++ ++++++++ free_open_stripe_run(bg, run);
+++++++++++++ ++ ++++++++ return tail_len;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Fold this block group's open-run remainder delta into the space_info
+++++++++++++ ++ ++++++++ * counter that reservations see. Bytes inside an open run are invisible
+++++++++++++ ++ ++++++++ * to the free space cache (the claim removed them) and will either be
+++++++++++++ ++ ++++++++ * allocated or become trapped tails when the run closes, so the data
+++++++++++++ ++ ++++++++ * reservation path must not admit writes against them: a reservation
+++++++++++++ ++ ++++++++ * that cannot allocate at writeback time discards the pages silently.
+++++++++++++ ++ ++++++++ * Called after every remainder mutation. Compute and apply are one
+++++++++++++ ++ ++++++++ * critical section (sinfo->lock nests inside stripe_run_lock): applied
+++++++++++++ ++ ++++++++ * out of order, a negative delta could overtake the positive it was
+++++++++++++ ++ ++++++++ * computed against and transiently underflow the space_info sum.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static void stripe_open_remainder_sync(struct btrfs_block_group *bg)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_space_info *sinfo = bg->space_info;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ s64 delta;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ delta = (s64)(bg->stripe_open_remainder - bg->stripe_open_reported);
+++++++++++++ ++ ++++++++ bg->stripe_open_reported = bg->stripe_open_remainder;
+++++++++++++ ++ ++++++++ if (delta && sinfo) {
+++++++++++++ ++ ++++++++ spin_lock(&sinfo->lock);
+++++++++++++ ++ ++++++++ if (delta < 0 && sinfo->bytes_stripe_open < (u64)-delta) {
+++++++++++++ ++ ++++++++ WARN_ON_ONCE(1);
+++++++++++++ ++ ++++++++ sinfo->bytes_stripe_open = 0;
+++++++++++++ ++ ++++++++ } else {
+++++++++++++ ++ ++++++++ sinfo->bytes_stripe_open += delta;
+++++++++++++ ++ ++++++++ }
++++++++++++++ +++++++++++ /*
++++++++++++++ +++++++++++ * A claimed run's remainder left the free space cache, but
++++++++++++++ +++++++++++ * reloc_reserve_free() still counts it: move it out of a held
++++++++++++++ +++++++++++ * group's share with it, or bytes_stripe_open and
++++++++++++++ +++++++++++ * bytes_reloc_reserve both count it until the next commit's
++++++++++++++ +++++++++++ * resync. Every writer of the share holds sinfo->lock.
++++++++++++++ +++++++++++ */
++++++++++++++ +++++++++++ btrfs_reloc_reserve_adjust(bg, -delta);
+++++++++++++ ++ ++++++++ spin_unlock(&sinfo->lock);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Install @run (not currently in any band slot) into its remaining-space band.
+++++++++++++ ++ ++++++++ * If a run already occupies that band, keep whichever has more remaining -- it
+++++++++++++ ++ ++++++++ * can serve larger future allocations, and closing the other traps less -- and
+++++++++++++ ++ ++++++++ * close the loser, returning its tail via *tail_start. Caller holds
+++++++++++++ ++ ++++++++ * bg->stripe_run_lock. Returns the closed tail length, or 0 if no collision.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static u64 stripe_run_place(struct btrfs_block_group *bg,
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run, u64 *tail_start)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ int class = run->class;
+++++++++++++ ++ ++++++++ int band = stripe_run_band(bg, run->end - run->offset);
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *occ = bg->open_stripe[class][band];
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Private runs (LOG, NOCOW, META_LOG) never occupy a band slot. */
+++++++++++++ ++ ++++++++ ASSERT(class != BTRFS_STRIPE_RUN_LOG);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (occ) {
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *keep, *shut;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (run->end - run->offset >= occ->end - occ->offset) {
+++++++++++++ ++ ++++++++ keep = run;
+++++++++++++ ++ ++++++++ shut = occ;
+++++++++++++ ++ ++++++++ } else {
+++++++++++++ ++ ++++++++ keep = occ;
+++++++++++++ ++ ++++++++ shut = run;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ bg->open_stripe[class][band] = keep;
+++++++++++++ ++ ++++++++ return close_open_stripe_run(bg, shut, tail_start);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ bg->open_stripe[class][band] = run;
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Allocate num_bytes from the block group's shared open runs of @class,
+++++++++++++ ++ ++++++++ * best fit across the remaining-space bands, claiming a new run via
+++++++++++++ ++ ++++++++ * btrfs_claim_free_stripe_run() when none fits. If the freshly claimed
+++++++++++++ ++ ++++++++ * stripes directly follow an open run of the same class, that run grows to
+++++++++++++ ++ ++++++++ * absorb them instead (see the comment at the claim site). The allocation
+++++++++++++ ++ ++++++++ * counts toward the run's inflight bytes until its IO is reported done or
+++++++++++++ ++ ++++++++ * abandoned.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in
+++++++++++++ ++ ++++++++ * this block group can satisfy the allocation, or -ENOMEM.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++int btrfs_alloc_from_open_stripe(struct btrfs_block_group *bg, u64 num_bytes,
+++++++++++++ ++ ++++++++ enum btrfs_stripe_run_class class,
+++++++++++++ ++ ++++++++ u64 *ret_offset, u64 *available)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++++ ++ ++++++++ const u64 transid = btrfs_get_fs_generation(fs_info);
+++++++++++++ ++ ++++++++ u64 evict_start[BTRFS_STRIPE_RUN_NR_BANDS];
+++++++++++++ ++ ++++++++ u64 evict_len[BTRFS_STRIPE_RUN_NR_BANDS];
+++++++++++++ ++ ++++++++ int nr_evict = 0;
+++++++++++++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *new_run;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ u64 tail_start = 0;
+++++++++++++ ++ ++++++++ u64 tail_len = 0;
+++++++++++++ ++ ++++++++ u64 start;
+++++++++++++ ++ ++++++++ u64 len;
+++++++++++++ ++ ++++++++ int band;
+++++++++++++ ++ ++++++++ int ret;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ ASSERT(num_bytes);
+++++++++++++ ++ ++++++++ *available = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Best fit: take the run in the smallest band that can hold this
+++++++++++++ ++ ++++++++ * allocation, and track the largest remaining across all bands as the
+++++++++++++ ++ ++++++++ * retry hint should nothing fit.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ run = NULL;
+++++++++++++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) {
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *r = bg->open_stripe[class][band];
+++++++++++++ ++ ++++++++ u64 rem;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!r)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Metadata runs are closed after the next transaction is
+++++++++++++ ++ ++++++++ * already running (btrfs_retire_meta_stripes()), so a slot can
+++++++++++++ ++ ++++++++ * still hold the previous transaction's run. Joining it would
+++++++++++++ ++ ++++++++ * put two transactions' tree blocks in the same stripes, and
+++++++++++++ ++ ++++++++ * the later one's writeback would read-modify-write parity
+++++++++++++ ++ ++++++++ * over the earlier one's committed blocks: the write hole.
+++++++++++++ ++ ++++++++ * Close it; its tail returns to the free space cache below.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_META && r->transid != transid) {
+++++++++++++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++++ ++ ++++++++ evict_len[nr_evict] =
+++++++++++++ ++ ++++++++ close_open_stripe_run(bg, r,
+++++++++++++ ++ ++++++++ &evict_start[nr_evict]);
+++++++++++++ ++ ++++++++ if (evict_len[nr_evict])
+++++++++++++ ++ ++++++++ nr_evict++;
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ rem = r->end - r->offset;
+++++++++++++ ++ ++++++++ if (rem > *available)
+++++++++++++ ++ ++++++++ *available = rem;
+++++++++++++ ++ ++++++++ if (!run && num_bytes <= rem) {
+++++++++++++ ++ ++++++++ run = r;
+++++++++++++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (run) {
+++++++++++++ ++ ++++++++ *ret_offset = run->offset;
+++++++++++++ ++ ++++++++ run_live_set(run, run->offset, num_bytes, true);
+++++++++++++ ++ ++++++++ run->offset += num_bytes;
+++++++++++++ ++ ++++++++ run->inflight_bytes += num_bytes;
+++++++++++++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
+++++++++++++ ++ ++++++++ if (run->offset == run->end)
+++++++++++++ ++ ++++++++ run->open = false; /* full: closed, no tail */
+++++++++++++ ++ ++++++++ else
+++++++++++++ ++ ++++++++ tail_len = stripe_run_place(bg, run, &tail_start);
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (tail_len)
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++++ ++ ++++++++ for (int i = 0; i < nr_evict; i++)
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, evict_start[i], evict_len[i]);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ for (int i = 0; i < nr_evict; i++)
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, evict_start[i], evict_len[i]);
+++++++++++++ ++ ++++++++ if (nr_evict)
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Zeroed, so the error paths can free ->live and ->written. */
+++++++++++++ ++ ++++++++ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
+++++++++++++ ++ ++++++++ if (!new_run)
+++++++++++++ ++ ++++++++ return -ENOMEM;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ ret = btrfs_claim_free_stripe_run(bg,
+++++++++++++ ++ ++++++++ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
+++++++++++++ ++ ++++++++ &start, &len);
+++++++++++++ ++ ++++++++ if (ret)
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ if (len < num_bytes) {
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Only a shorter run is free. Put it back untouched (it is
+++++++++++++ ++ ++++++++ * still fully-free stripes) and let the caller retry with a
+++++++++++++ ++ ++++++++ * smaller allocation.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++++ ++ ++++++++ *available = max(*available, len);
+++++++++++++ ++ ++++++++ ret = -ENOSPC;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Sleeps, so it has to happen before the locks below. */
+++++++++++++ ++ ++++++++ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
+++++++++++++ ++ ++++++++ new_run->written = new_run->live_nbits ?
+++++++++++++ ++ ++++++++ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Install under fs_info->open_stripe_lock so that the block group
+++++++++++++ ++ ++++++++ * membership, the retire sequence read and the run installation are
+++++++++++++ ++ ++++++++ * one atomic step against the commit's retire walk (which bumps the
+++++++++++++ ++ ++++++++ * sequence and snapshots the membership list under the same lock).
+++++++++++++ ++ ++++++++ * A run is therefore either visible to the walk that retires its
+++++++++++++ ++ ++++++++ * window, or stamped with a sequence that walk's transaction did
+++++++++++++ ++ ++++++++ * not retire, which defers its extents to the next transaction.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ if (list_empty(&bg->open_stripe_bg_list)) {
+++++++++++++ ++ ++++++++ btrfs_get_block_group(bg);
+++++++++++++ ++ ++++++++ list_add_tail(&bg->open_stripe_bg_list,
+++++++++++++ ++ ++++++++ &fs_info->open_stripe_bgs);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (bg->ro) {
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Lost a race with this block group turning read-only; run
+++++++++++++ ++ ++++++++ * retirement for it may already have happened, so a new run
+++++++++++++ ++ ++++++++ * must not be installed.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++++ ++ ++++++++ *available = 0;
+++++++++++++ ++ ++++++++ ret = -ENOSPC;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Growable frontier: if the claimed region directly follows an open run
+++++++++++++ ++ ++++++++ * of the same class, grow that run over it and place this allocation at
+++++++++++++ ++ ++++++++ * the run's old tail, so the tail is used rather than stranded when the
+++++++++++++ ++ ++++++++ * run closes. Stripe exclusivity holds: the grown stripes were claimed
+++++++++++++ ++ ++++++++ * fully free, and the run is still retired at commit. The run keeps
+++++++++++++ ++ ++++++++ * its open_seq: it can only still be in a band slot if the retire walk
+++++++++++++ ++ ++++++++ * for its window has not run, which this lock nesting excludes.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) {
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *r = bg->open_stripe[class][band];
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!r || r->end != start)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /* Never grow another transaction's run (see the scan above). */
+++++++++++++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_META && r->transid != transid)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Only grow as far as the run's liveness map reaches; it
+++++++++++++ ++ ++++++++ * cannot be reallocated here (see alloc_run_live_map()).
+++++++++++++ ++ ++++++++ * Beyond that, leave the run alone and open a new one.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (((start + len - r->start) >> fs_info->sectorsize_bits) >
+++++++++++++ ++ ++++++++ r->live_nbits)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ ASSERT(r->open);
+++++++++++++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++++ ++ ++++++++ *ret_offset = r->offset;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Mark the handed-out bytes live before anything can consult
+++++++++++++ ++ ++++++++ * the map: this allocation begins at the old tail, usually
+++++++++++++ ++ ++++++++ * mid-stripe, and a sector the map calls dead is a sector a
+++++++++++++ ++ ++++++++ * neighbour's padded write may zero-fill.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ run_live_set(r, r->offset, num_bytes, true);
+++++++++++++ ++ ++++++++ r->end = start + len;
+++++++++++++ ++ ++++++++ r->offset += num_bytes;
+++++++++++++ ++ ++++++++ r->inflight_bytes += num_bytes;
+++++++++++++ ++ ++++++++ bg->stripe_open_remainder += len;
+++++++++++++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
+++++++++++++ ++ ++++++++ if (r->offset == r->end)
+++++++++++++ ++ ++++++++ r->open = false; /* full: closed, no tail */
+++++++++++++ ++ ++++++++ else
+++++++++++++ ++ ++++++++ tail_len = stripe_run_place(bg, r, &tail_start);
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ if (tail_len)
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ ret = 0;
+++++++++++++ ++ ++++++++ goto out; /* frees the unused new_run */
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ new_run->bg = bg;
+++++++++++++ ++ ++++++++ new_run->class = class;
+++++++++++++ ++ ++++++++ new_run->persistent = false;
+++++++++++++ ++ ++++++++ new_run->retained = class == BTRFS_STRIPE_RUN_RELOC;
+++++++++++++ ++ ++++++++ if (new_run->retained)
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.reloc_run_retained);
+++++++++++++ ++ ++++++++ new_run->owner = 0;
+++++++++++++ ++ ++++++++ new_run->transid = transid;
+++++++++++++ ++ ++++++++ new_run->start = start;
+++++++++++++ ++ ++++++++ new_run->end = start + len;
+++++++++++++ ++ ++++++++ new_run->offset = start + num_bytes;
+++++++++++++ ++ ++++++++ new_run->inflight_bytes = num_bytes;
+++++++++++++ ++ ++++++++ new_run->open_seq = fs_info->stripe_retire_seq;
+++++++++++++ ++ ++++++++ new_run->open = (new_run->offset != new_run->end);
+++++++++++++ ++ ++++++++ run_live_set(new_run, start, num_bytes, true);
+++++++++++++ ++ ++++++++ list_add_tail(&new_run->list, &bg->open_stripe_runs);
+++++++++++++ ++ ++++++++ bg->stripe_open_remainder += new_run->end - new_run->offset;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Place the new run in its remaining-space band, closing whichever run
+++++++++++++ ++ ++++++++ * loses a band collision (the last, fuller opener wins). Other bands'
+++++++++++++ ++ ++++++++ * open runs are untouched.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (new_run->open)
+++++++++++++ ++ ++++++++ tail_len = stripe_run_place(bg, new_run, &tail_start);
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (tail_len)
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ *ret_offset = start;
+++++++++++++ ++ ++++++++ new_run = NULL;
+++++++++++++ ++ ++++++++ ret = 0;
+++++++++++++ ++ ++++++++out:
+++++++++++++ ++ ++++++++ /* Unused on every path that did not install it, including errors. */
+++++++++++++ ++ ++++++++ if (new_run) {
+++++++++++++ ++ ++++++++ bitmap_free(new_run->live);
+++++++++++++ ++ ++++++++ bitmap_free(new_run->written);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ kfree(new_run);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Allocate a log tree block from the private run of this log commit.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * The run is keyed by @owner (the subvolume's root id) and @log_transid, so
+++++++++++++ ++ ++++++++ * each log commit gets whole stripes of its own: its blocks are durable once
+++++++++++++ ++ ++++++++ * fsync returns, and no later writer may share a stripe with them (see
+++++++++++++ ++ ++++++++ * BTRFS_STRIPE_RUN_META_LOG). An open run of the same owner from an earlier
+++++++++++++ ++ ++++++++ * log commit is closed, as a stale-transid metadata run is in
+++++++++++++ ++ ++++++++ * btrfs_alloc_from_open_stripe(). Like the other private runs, it is never
+++++++++++++ ++ ++++++++ * placed in a band slot.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in this
+++++++++++++ ++ ++++++++ * block group can satisfy the allocation, or -ENOMEM.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++int btrfs_alloc_from_log_stripe_run(struct btrfs_block_group *bg, u64 num_bytes,
+++++++++++++ ++ ++++++++ u64 owner, u64 log_transid,
+++++++++++++ ++ ++++++++ u64 *ret_offset, u64 *available)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++++ ++ ++++++++ const enum btrfs_stripe_run_class class = BTRFS_STRIPE_RUN_META_LOG;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *new_run;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ u64 tail_start = 0;
+++++++++++++ ++ ++++++++ u64 tail_len = 0;
+++++++++++++ ++ ++++++++ u64 start;
+++++++++++++ ++ ++++++++ u64 len;
+++++++++++++ ++ ++++++++ int ret;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ ASSERT(num_bytes);
+++++++++++++ ++ ++++++++ *available = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (run->class != class || run->owner != owner || !run->open)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * A previous log commit of the same subvolume. Its blocks are
+++++++++++++ ++ ++++++++ * durable, so this one must not continue into them; close it
+++++++++++++ ++ ++++++++ * and claim fresh stripes below. log_transid starts over with
+++++++++++++ ++ ++++++++ * every transaction's log tree, so a run from an earlier
+++++++++++++ ++ ++++++++ * transaction can carry the same log_transid: the transid
+++++++++++++ ++ ++++++++ * tells them apart. Joining it would hand that transaction's
+++++++++++++ ++ ++++++++ * commit, which retires the run and waits for its blocks, a
+++++++++++++ ++ ++++++++ * block only this transaction will write.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (run->log_transid != log_transid ||
+++++++++++++ ++ ++++++++ run->transid != btrfs_get_fs_generation(fs_info)) {
+++++++++++++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (num_bytes <= run->end - run->offset) {
+++++++++++++ ++ ++++++++ *ret_offset = run->offset;
+++++++++++++ ++ ++++++++ run_live_set(run, run->offset, num_bytes, true);
+++++++++++++ ++ ++++++++ run->offset += num_bytes;
+++++++++++++ ++ ++++++++ run->inflight_bytes += num_bytes;
+++++++++++++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
+++++++++++++ ++ ++++++++ if (run->offset == run->end)
+++++++++++++ ++ ++++++++ run->open = false; /* full: closed, no tail */
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ /* Too small for this block: replace it. */
+++++++++++++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (tail_len) {
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Zeroed: the error paths below free ->live and must find NULL. */
+++++++++++++ ++ ++++++++ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
+++++++++++++ ++ ++++++++ if (!new_run)
+++++++++++++ ++ ++++++++ return -ENOMEM;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ ret = btrfs_claim_free_stripe_run(bg,
+++++++++++++ ++ ++++++++ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
+++++++++++++ ++ ++++++++ &start, &len);
+++++++++++++ ++ ++++++++ if (ret)
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ if (len < num_bytes) {
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++++ ++ ++++++++ *available = max(*available, len);
+++++++++++++ ++ ++++++++ ret = -ENOSPC;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Sleeps, so it has to happen before the locks below. */
+++++++++++++ ++ ++++++++ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
+++++++++++++ ++ ++++++++ new_run->written = new_run->live_nbits ?
+++++++++++++ ++ ++++++++ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ if (list_empty(&bg->open_stripe_bg_list)) {
+++++++++++++ ++ ++++++++ btrfs_get_block_group(bg);
+++++++++++++ ++ ++++++++ list_add_tail(&bg->open_stripe_bg_list,
+++++++++++++ ++ ++++++++ &fs_info->open_stripe_bgs);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (bg->ro) {
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++++ ++ ++++++++ *available = 0;
+++++++++++++ ++ ++++++++ ret = -ENOSPC;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ new_run->bg = bg;
+++++++++++++ ++ ++++++++ new_run->class = class;
+++++++++++++ ++ ++++++++ new_run->persistent = false;
+++++++++++++ ++ ++++++++ new_run->root = owner;
+++++++++++++ ++ ++++++++ new_run->owner = owner;
+++++++++++++ ++ ++++++++ new_run->log_transid = log_transid;
+++++++++++++ ++ ++++++++ new_run->transid = btrfs_get_fs_generation(fs_info);
+++++++++++++ ++ ++++++++ new_run->start = start;
+++++++++++++ ++ ++++++++ new_run->end = start + len;
+++++++++++++ ++ ++++++++ new_run->offset = start + num_bytes;
+++++++++++++ ++ ++++++++ new_run->inflight_bytes = num_bytes;
+++++++++++++ ++ ++++++++ new_run->open_seq = fs_info->stripe_retire_seq;
+++++++++++++ ++ ++++++++ new_run->open = (new_run->offset != new_run->end);
+++++++++++++ ++ ++++++++ run_live_set(new_run, start, num_bytes, true);
+++++++++++++ ++ ++++++++ list_add_tail(&new_run->list, &bg->open_stripe_runs);
+++++++++++++ ++ ++++++++ bg->stripe_open_remainder += new_run->end - new_run->offset;
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.log_run_opened);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ *ret_offset = start;
+++++++++++++ ++ ++++++++ new_run = NULL;
+++++++++++++ ++ ++++++++ ret = 0;
+++++++++++++ ++ ++++++++out:
+++++++++++++ ++ ++++++++ if (new_run) {
+++++++++++++ ++ ++++++++ bitmap_free(new_run->live);
+++++++++++++ ++ ++++++++ bitmap_free(new_run->written);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ kfree(new_run);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * The owner a private run is found by: the inode, or with
+++++++++++++ ++ ++++++++ * stripe_log_data_shared a LOG-class run shared by every inode of the
+++++++++++++ ++ ++++++++ * subvolume logging in the same log transaction. An allocation of a full
+++++++++++++ ++ ++++++++ * stripe or more keeps the inode's private run, so small fsync extents never
+++++++++++++ ++ ++++++++ * share the partial last stripe of a large one and wait for its IO. Bit 63
+++++++++++++ ++ ++++++++ * keeps the shared keys apart from inode numbers; the subvolume is matched
+++++++++++++ ++ ++++++++ * separately, by run->root.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static u64 stripe_run_owner(struct btrfs_inode *inode,
+++++++++++++ ++ ++++++++ enum btrfs_stripe_run_class class, u64 num_bytes,
+++++++++++++ ++ ++++++++ u64 fsl)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_root *root = inode->root;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_LOG && num_bytes < fsl &&
+++++++++++++ ++ ++++++++ READ_ONCE(root->fs_info->stripe_log_data_shared))
+++++++++++++ ++ ++++++++ return BIT_ULL(63) | (u32)READ_ONCE(root->log_transid);
+++++++++++++ ++ ++++++++ return btrfs_ino(inode);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Allocate num_bytes for @inode from its private run of @class (LOG or
+++++++++++++ ++ ++++++++ * NOCOW) in this block group, claiming a fresh run when there is none or it
+++++++++++++ ++ ++++++++ * does not fit. With stripe_log_data_shared, small LOG allocations use the
+++++++++++++ ++ ++++++++ * run the subvolume shares for the current log transaction
+++++++++++++ ++ ++++++++ * (stripe_run_owner()). Private runs are never placed in a band slot and
+++++++++++++ ++ ++++++++ * are found by subvolume and owner, so a log commit settles only stripes
+++++++++++++ ++ ++++++++ * holding its own extents. A NOCOW allocation may instead re-adopt the
+++++++++++++ ++ ++++++++ * free tail of the inode's last partial stripe (@readopt_hint).
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * A run too small for this allocation is closed at once rather than left
+++++++++++++ ++ ++++++++ * to strand its tail until commit. Two racing allocations for one owner
+++++++++++++ ++ ++++++++ * may each claim a run; the spare drains at the next commit, costing a
+++++++++++++ ++ ++++++++ * trapped tail, not correctness.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in
+++++++++++++ ++ ++++++++ * this block group can satisfy the allocation, or -ENOMEM.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++int btrfs_alloc_from_inode_stripe_run(struct btrfs_block_group *bg,
+++++++++++++ ++ ++++++++ struct btrfs_inode *inode,
+++++++++++++ ++ ++++++++ enum btrfs_stripe_run_class class,
+++++++++++++ ++ ++++++++ u64 num_bytes, u64 readopt_hint,
+++++++++++++ ++ ++++++++ u64 *ret_offset, u64 *available)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++++ ++ ++++++++ const u64 root_id = btrfs_root_id(inode->root);
+++++++++++++ ++ ++++++++ const u64 ino = stripe_run_owner(inode, class, num_bytes, fsl);
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *new_run;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ u64 tail_start = 0;
+++++++++++++ ++ ++++++++ u64 tail_len = 0;
+++++++++++++ ++ ++++++++ u64 start;
+++++++++++++ ++ ++++++++ u64 len;
+++++++++++++ ++ ++++++++ int ret;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ ASSERT(num_bytes);
+++++++++++++ ++ ++++++++ *available = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (run->class != class || run->root != root_id ||
+++++++++++++ ++ ++++++++ run->owner != ino || !run->open)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (num_bytes <= run->end - run->offset) {
+++++++++++++ ++ ++++++++ *ret_offset = run->offset;
+++++++++++++ ++ ++++++++ run_live_set(run, run->offset, num_bytes, true);
+++++++++++++ ++ ++++++++ run->offset += num_bytes;
+++++++++++++ ++ ++++++++ run->inflight_bytes += num_bytes;
+++++++++++++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
+++++++++++++ ++ ++++++++ if (run->offset == run->end)
+++++++++++++ ++ ++++++++ run->open = false; /* full: closed, no tail */
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (tail_len) {
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Zeroed, so the error paths can free ->live and ->written. */
+++++++++++++ ++ ++++++++ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
+++++++++++++ ++ ++++++++ if (!new_run)
+++++++++++++ ++ ++++++++ return -ENOMEM;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Re-adoption (nocow only): after a remount (or eviction) the
+++++++++++++ ++ ++++++++ * inode's last partial stripe has no run, and a fresh claim would
+++++++++++++ ++ ++++++++ * abandon its free tail forever. When the allocation hint points
+++++++++++++ ++ ++++++++ * into a stripe wholly owned by this inode, adopt the tail as the
+++++++++++++ ++ ++++++++ * private run instead: every byte below the frontier is provably
+++++++++++++ ++ ++++++++ * the inode's own committed data, and nothing else can consume a
+++++++++++++ ++ ++++++++ * partial stripe's free space (see btrfs_claim_stripe_tail()).
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_NOCOW && readopt_hint &&
+++++++++++++ ++ ++++++++ readopt_hint >= bg->start &&
+++++++++++++ ++ ++++++++ readopt_hint < bg->start + bg->length) {
+++++++++++++ ++ ++++++++ u64 stripe = bg->start +
+++++++++++++ ++ ++++++++ stripe_round_down(readopt_hint - bg->start, fsl);
+++++++++++++ ++ ++++++++ u64 frontier = 0;
+++++++++++++ ++ ++++++++ u64 tlen;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Under space_info->groups_sem (we are inside
+++++++++++++ ++ ++++++++ * find_free_extent()), so the lookup must not block -- see
+++++++++++++ ++ ++++++++ * stripe_extents_owned_by(). A trylock failure just declines
+++++++++++++ ++ ++++++++ * the re-adoption and falls through to a fresh claim below,
+++++++++++++ ++ ++++++++ * costing this stripe's free tail, not correctness.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (stripe_extents_owned_by(fs_info, root_id, ino, stripe, fsl,
+++++++++++++ ++ ++++++++ &frontier, true) &&
+++++++++++++ ++ ++++++++ frontier > stripe && frontier < stripe + fsl &&
+++++++++++++ ++ ++++++++ num_bytes <= stripe + fsl - frontier) {
+++++++++++++ ++ ++++++++ tlen = stripe + fsl - frontier;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * btrfs_stripe_run_range_usable() is for whole-stripe
+++++++++++++ ++ ++++++++ * claims (it rejects anything shorter than a full
+++++++++++++ ++ ++++++++ * stripe); a tail needs only the run-overlap check.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (stripe_range_run_free(bg, frontier, tlen) &&
+++++++++++++ ++ ++++++++ !btrfs_claim_stripe_tail(bg, frontier, tlen)) {
+++++++++++++ ++ ++++++++ start = frontier;
+++++++++++++ ++ ++++++++ len = tlen;
+++++++++++++ ++ ++++++++ goto install;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ ret = btrfs_claim_free_stripe_run(bg,
+++++++++++++ ++ ++++++++ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
+++++++++++++ ++ ++++++++ &start, &len);
+++++++++++++ ++ ++++++++ if (ret)
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ if (len < num_bytes) {
+++++++++++++ ++ ++++++++ /* Only a shorter run is free; see btrfs_alloc_from_open_stripe(). */
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++++ ++ ++++++++ *available = len;
+++++++++++++ ++ ++++++++ ret = -ENOSPC;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++install:
+++++++++++++ ++ ++++++++ /* Sleeps, so it has to happen before the locks below. */
+++++++++++++ ++ ++++++++ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
+++++++++++++ ++ ++++++++ new_run->written = new_run->live_nbits ?
+++++++++++++ ++ ++++++++ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* See the installation comment in btrfs_alloc_from_open_stripe(). */
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ if (list_empty(&bg->open_stripe_bg_list)) {
+++++++++++++ ++ ++++++++ btrfs_get_block_group(bg);
+++++++++++++ ++ ++++++++ list_add_tail(&bg->open_stripe_bg_list,
+++++++++++++ ++ ++++++++ &fs_info->open_stripe_bgs);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (bg->ro) {
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++++ ++ ++++++++ *available = 0;
+++++++++++++ ++ ++++++++ ret = -ENOSPC;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ new_run->bg = bg;
+++++++++++++ ++ ++++++++ new_run->class = class;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * A NOCOW run outlives commits only while the stripe_alloc_allow_rmw
+++++++++++++ ++ ++++++++ * policy lets nocow data be written in place at all (see
+++++++++++++ ++ ++++++++ * close_block_group_stripe_runs()). Decided once, here: the commit
+++++++++++++ ++ ++++++++ * path's close and settle predicates must agree with each other for
+++++++++++++ ++ ++++++++ * the run's whole life, whatever the policy does meanwhile.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ new_run->persistent = class == BTRFS_STRIPE_RUN_NOCOW &&
+++++++++++++ ++ ++++++++ (btrfs_stripe_allow_rmw(fs_info) &
+++++++++++++ ++ ++++++++ (BTRFS_STRIPE_RMW_NODATACOW |
+++++++++++++ ++ ++++++++ BTRFS_STRIPE_RMW_PREALLOC));
+++++++++++++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_NOCOW)
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.nocow_run_opened);
+++++++++++++ ++ ++++++++ new_run->root = root_id;
+++++++++++++ ++ ++++++++ new_run->owner = ino;
+++++++++++++ ++ ++++++++ new_run->start = start;
+++++++++++++ ++ ++++++++ new_run->end = start + len;
+++++++++++++ ++ ++++++++ new_run->offset = start + num_bytes;
+++++++++++++ ++ ++++++++ new_run->inflight_bytes = num_bytes;
+++++++++++++ ++ ++++++++ new_run->open_seq = fs_info->stripe_retire_seq;
+++++++++++++ ++ ++++++++ new_run->open = (new_run->offset != new_run->end);
+++++++++++++ ++ ++++++++ run_live_set(new_run, start, num_bytes, true);
+++++++++++++ ++ ++++++++ list_add_tail(&new_run->list, &bg->open_stripe_runs);
+++++++++++++ ++ ++++++++ bg->stripe_open_remainder += new_run->end - new_run->offset;
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ *ret_offset = start;
+++++++++++++ ++ ++++++++ new_run = NULL;
+++++++++++++ ++ ++++++++ ret = 0;
+++++++++++++ ++ ++++++++out:
+++++++++++++ ++ ++++++++ /* Unused on every path that did not install it, including errors. */
+++++++++++++ ++ ++++++++ if (new_run) {
+++++++++++++ ++ ++++++++ bitmap_free(new_run->live);
+++++++++++++ ++ ++++++++ bitmap_free(new_run->written);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ kfree(new_run);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * An extent inside a stripe run's range has been freed: its sectors no longer
+++++++++++++ ++ ++++++++ * hold data anyone intends to write, so clear their live bits.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * This is what makes a run detectably dead (open_stripe_run_dead()), so the
+++++++++++++ ++ ++++++++ * next commit can close it and release its stripes. Without it a nodatacow
+++++++++++++ ++ ++++++++ * or preallocating inode's private run hoards its stripes until the inode is
+++++++++++++ ++ ++++++++ * evicted. A partial write to one of those stripes can also pad over the
+++++++++++++ ++ ++++++++ * freed sectors now instead of falling back to a read-modify-write.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Takes only bg->stripe_run_lock, and the callers hold no free space lock
+++++++++++++ ++ ++++++++ * yet, so no nesting rule applies. The range is clamped to the run: a free
+++++++++++++ ++ ++++++++ * may span a run's boundary, and running off the live map would retire it.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_stripe_run_note_free(struct btrfs_block_group *bg, u64 bytenr,
+++++++++++++ ++ ++++++++ u64 size)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!btrfs_is_stripe_alloc_bg(bg))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ u64 start = max(bytenr, run->start);
+++++++++++++ ++ ++++++++ u64 end = min(bytenr + size, run->offset);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Only the persistent runs, which are the only ones that can
+++++++++++++ ++ ++++++++ * outlive their own data. Every other class is closed and
+++++++++++++ ++ ++++++++ * freed by the commit it belongs to, so its live map is never
+++++++++++++ ++ ++++++++ * consulted after its bytes could have been freed, and
+++++++++++++ ++ ++++++++ * clearing bits there would change padding decisions for no
+++++++++++++ ++ ++++++++ * gain here.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (!run->persistent)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (end <= start)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ run_live_set(run, start, end - start, false);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * A run's range must not be claimed again while the run object exists, even
+++++++++++++ ++ ++++++++ * if every byte of one of its stripes is back in the free space cache
+++++++++++++ ++ ++++++++ * (abandoned allocations): a second run over the same range would make the
+++++++++++++ ++ ++++++++ * range-to-run lookups (ordered extent attach, IO reporting) ambiguous and
+++++++++++++ ++ ++++++++ * corrupt the inflight accounting. Trim the candidate to end before the
+++++++++++++ ++ ++++++++ * first overlapping run, and reject it if its head overlaps or less than a
+++++++++++++ ++ ++++++++ * full stripe remains. Caller holds ctl->tree_lock; stripe_run_lock nests
+++++++++++++ ++ ++++++++ * inside it.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++bool btrfs_stripe_run_range_usable(struct btrfs_block_group *bg,
+++++++++++++ ++ ++++++++ u64 run_start, u64 *run_len)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ u64 len = *run_len;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (run->end <= run_start || run->start >= run_start + len)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (run->start <= run_start) {
+++++++++++++ ++ ++++++++ len = 0;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ len = run->start - run_start;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (len < bg->full_stripe_len) {
+++++++++++++ ++ ++++++++ atomic64_inc(&bg->fs_info->stripe_run_stats.claim_refused_live_run);
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ *run_len = len;
+++++++++++++ ++ ++++++++ return true;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * For the stripe_unusable scan: whole free stripes inside a live run's
+++++++++++++ ++ ++++++++ * range cannot be claimed while the run exists
+++++++++++++ ++ ++++++++ * (btrfs_stripe_run_range_usable()), so they must not count as claimable
+++++++++++++ ++ ++++++++ * or the admission gate would admit writes that cannot be placed. Clear
+++++++++++++ ++ ++++++++ * them from @freep and return their total; free_open_stripe_run() marks the
+++++++++++++ ++ ++++++++ * group for rescan so they are credited back when the run goes. Caller
+++++++++++++ ++ ++++++++ * holds ctl->tree_lock; stripe_run_lock nests inside it.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++u64 btrfs_stripe_run_stranded_stripes(struct btrfs_block_group *bg, u32 *freep,
+++++++++++++ ++ ++++++++ u64 nstripes)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ u64 stranded = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ u64 idx, last;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (run->end <= run->start)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ idx = div64_u64(run->start - bg->start, fsl);
+++++++++++++ ++ ++++++++ last = div64_u64(run->end - 1 - bg->start, fsl);
+++++++++++++ ++ ++++++++ for (; idx <= last && idx < nstripes; idx++) {
+++++++++++++ ++ ++++++++ if (freep[idx] != fsl)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ freep[idx] = 0;
+++++++++++++ ++ ++++++++ stranded += fsl;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (stranded)
+++++++++++++ ++ ++++++++ atomic64_add(stranded, &bg->fs_info->stripe_run_stats.strand_scan_bytes);
+++++++++++++ ++ ++++++++ return stranded;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Is the full stripe at @stripe inside a live run's range, i.e. stranded by
+++++++++++++ ++ ++++++++ * the rule of btrfs_stripe_run_stranded_stripes()? Caller holds
+++++++++++++ ++ ++++++++ * ctl->tree_lock.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++bool btrfs_stripe_run_strands(struct btrfs_block_group *bg, u64 stripe)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ bool ret = false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (run->end > run->start &&
+++++++++++++ ++ ++++++++ stripe < run->end && stripe + fsl > run->start) {
+++++++++++++ ++ ++++++++ ret = true;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Parse the value of the btrfs.stripe_alloc_allow_rmw property, a word list
+++++++++++++ ++ ++++++++ * ("nodatacow", "prealloc", "fsync"), into its mask. Words are separated by
+++++++++++++ ++ ++++++++ * comma, space or colon. An empty value yields an empty mask. Returns
+++++++++++++ ++ ++++++++ * -EINVAL on any unknown word.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++int btrfs_parse_stripe_rmw(const char *value, size_t len, u32 *mask)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ u32 m = 0;
+++++++++++++ ++ ++++++++ size_t i = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ while (i < len) {
+++++++++++++ ++ ++++++++ size_t j = i;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ while (j < len && value[j] != ',' && value[j] != ' ' &&
+++++++++++++ ++ ++++++++ value[j] != ':')
+++++++++++++ ++ ++++++++ j++;
+++++++++++++ ++ ++++++++ if (j - i == strlen("nodatacow") &&
+++++++++++++ ++ ++++++++ !strncmp(value + i, "nodatacow", j - i))
+++++++++++++ ++ ++++++++ m |= BTRFS_STRIPE_RMW_NODATACOW;
+++++++++++++ ++ ++++++++ else if (j - i == strlen("prealloc") &&
+++++++++++++ ++ ++++++++ !strncmp(value + i, "prealloc", j - i))
+++++++++++++ ++ ++++++++ m |= BTRFS_STRIPE_RMW_PREALLOC;
+++++++++++++ ++ ++++++++ else if (j - i == strlen("fsync") &&
+++++++++++++ ++ ++++++++ !strncmp(value + i, "fsync", j - i))
+++++++++++++ ++ ++++++++ m |= BTRFS_STRIPE_RMW_FSYNC;
+++++++++++++ ++ ++++++++ else if (j != i)
+++++++++++++ ++ ++++++++ return -EINVAL;
+++++++++++++ ++ ++++++++ i = j + 1;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ *mask = m;
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Does every committed extent in [start, start + len) belong solely to
+++++++++++++ ++ ++++++++ * @ino in @root_id -- exactly one reference, a plain data ref, count 1?
+++++++++++++ ++ ++++++++ * Any metadata block, shared reference, or foreign owner fails. Caller
+++++++++++++ ++ ++++++++ * limits the range to full stripes; uncommitted allocations need not be
+++++++++++++ ++ ++++++++ * visible because nothing can allocate into a partially used stripe, so a
+++++++++++++ ++ ++++++++ * foreign extent can only appear after the whole stripe frees and commits.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Callers holding space_info->groups_sem (anything reached from
+++++++++++++ ++ ++++++++ * find_free_extent()) must pass @nowait: delayed ref processing takes
+++++++++++++ ++ ++++++++ * extent tree locks and then groups_sem, so blocking on an extent tree lock
+++++++++++++ ++ ++++++++ * here can deadlock behind a queued groups_sem writer. Callers that must
+++++++++++++ ++ ++++++++ * not block for other reasons pass it too. With @nowait a contended lock
+++++++++++++ ++ ++++++++ * fails the search and the answer is false, which every caller treats only
+++++++++++++ ++ ++++++++ * as a reason to skip an optimisation.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static bool stripe_extents_owned_by(struct btrfs_fs_info *fs_info,
+++++++++++++ ++ ++++++++ u64 root_id, u64 ino, u64 start, u64 len,
+++++++++++++ ++ ++++++++ u64 *frontier, bool nowait)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_root *extent_root = btrfs_extent_root(fs_info, start);
+++++++++++++ ++ ++++++++ struct btrfs_path *path;
+++++++++++++ ++ ++++++++ struct btrfs_key key;
+++++++++++++ ++ ++++++++ bool ret = true;
+++++++++++++ ++ ++++++++ int err;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!extent_root)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ path = btrfs_alloc_path();
+++++++++++++ ++ ++++++++ if (!path)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Read-only search, so nowait is allowed, and every step of the walk
+++++++++++++ ++ ++++++++ * below honours it (btrfs_next_leaf() runs with time_seq 0).
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ path->nowait = nowait;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ key.objectid = start;
+++++++++++++ ++ ++++++++ key.type = BTRFS_EXTENT_ITEM_KEY;
+++++++++++++ ++ ++++++++ key.offset = 0;
+++++++++++++ ++ ++++++++ err = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+++++++++++++ ++ ++++++++ if (err < 0) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ /* An extent starting before @start may still overlap it. */
+++++++++++++ ++ ++++++++ err = btrfs_previous_extent_item(extent_root, path, 0);
+++++++++++++ ++ ++++++++ if (err < 0) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (err > 0) {
+++++++++++++ ++ ++++++++ btrfs_release_path(path);
+++++++++++++ ++ ++++++++ key.objectid = start;
+++++++++++++ ++ ++++++++ key.type = BTRFS_EXTENT_ITEM_KEY;
+++++++++++++ ++ ++++++++ key.offset = 0;
+++++++++++++ ++ ++++++++ err = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+++++++++++++ ++ ++++++++ if (err < 0) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ while (true) {
+++++++++++++ ++ ++++++++ struct extent_buffer *leaf = path->nodes[0];
+++++++++++++ ++ ++++++++ int slot = path->slots[0];
+++++++++++++ ++ ++++++++ struct btrfs_extent_item *ei;
+++++++++++++ ++ ++++++++ struct btrfs_extent_inline_ref *iref;
+++++++++++++ ++ ++++++++ struct btrfs_extent_data_ref *dref;
+++++++++++++ ++ ++++++++ int type;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (slot >= btrfs_header_nritems(leaf)) {
+++++++++++++ ++ ++++++++ err = btrfs_next_leaf(extent_root, path);
+++++++++++++ ++ ++++++++ if (err < 0) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (err > 0)
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ btrfs_item_key_to_cpu(leaf, &key, slot);
+++++++++++++ ++ ++++++++ if (key.objectid >= start + len)
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ if (key.type == BTRFS_METADATA_ITEM_KEY) {
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * key.offset is the level, not a length; a tree
+++++++++++++ ++ ++++++++ * block spans nodesize bytes. One from before the
+++++++++++++ ++ ++++++++ * range (e.g. where the pre-range positioning landed
+++++++++++++ ++ ++++++++ * on a metadata block group's last item) is harmless.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (key.objectid + fs_info->nodesize > start) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ path->slots[0]++;
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (key.type != BTRFS_EXTENT_ITEM_KEY ||
+++++++++++++ ++ ++++++++ key.objectid + key.offset <= start) {
+++++++++++++ ++ ++++++++ path->slots[0]++;
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item);
+++++++++++++ ++ ++++++++ if (btrfs_extent_refs(leaf, ei) != 1 ||
+++++++++++++ ++ ++++++++ !(btrfs_extent_flags(leaf, ei) & BTRFS_EXTENT_FLAG_DATA)) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * With a keyed (non-inline) backref the item ends after the
+++++++++++++ ++ ++++++++ * header, and reading an inline ref would read the next item.
+++++++++++++ ++ ++++++++ * Treat it as not owned, the conservative answer.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (btrfs_item_size(leaf, slot) < sizeof(*ei) +
+++++++++++++ ++ ++++++++ btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY)) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ iref = (struct btrfs_extent_inline_ref *)(ei + 1);
+++++++++++++ ++ ++++++++ type = btrfs_get_extent_inline_ref_type(leaf, iref,
+++++++++++++ ++ ++++++++ BTRFS_REF_TYPE_DATA);
+++++++++++++ ++ ++++++++ if (type != BTRFS_EXTENT_DATA_REF_KEY) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ dref = (struct btrfs_extent_data_ref *)(&iref->offset);
+++++++++++++ ++ ++++++++ if (btrfs_extent_data_ref_root(leaf, dref) != root_id ||
+++++++++++++ ++ ++++++++ btrfs_extent_data_ref_objectid(leaf, dref) != ino ||
+++++++++++++ ++ ++++++++ btrfs_extent_data_ref_count(leaf, dref) != 1) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (frontier && key.objectid + key.offset > *frontier)
+++++++++++++ ++ ++++++++ *frontier = min(key.objectid + key.offset, start + len);
+++++++++++++ ++ ++++++++ path->slots[0]++;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++out:
+++++++++++++ ++ ++++++++ btrfs_free_path(path);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * May [start, start + num_bytes) be written in place under stripe-exclusive
+++++++++++++ ++ ++++++++ * allocation? Only when stripe_alloc_allow_rmw includes @which and every
+++++++++++++ ++ ++++++++ * full stripe the range touches is isolated to @inode: an in-place write
+++++++++++++ ++ ++++++++ * RMWs parity, so a crash can tear every extent sharing the stripe, and
+++++++++++++ ++ ++++++++ * isolation confines that to the file that opted out of COW. Fast path:
+++++++++++++ ++ ++++++++ * the stripe is in one of the inode's own NOCOW runs. Slow path: the
+++++++++++++ ++ ++++++++ * committed extent tree proves sole ownership. Extents that fail stay
+++++++++++++ ++ ++++++++ * COWed, and their rewrite lands in the inode's NOCOW run, so the next
+++++++++++++ ++ ++++++++ * overwrite passes.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++bool btrfs_stripe_nocow_writable(struct btrfs_inode *inode, u64 start,
+++++++++++++ ++ ++++++++ u64 num_bytes, u32 which, bool nowait)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = inode->root->fs_info;
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ const u64 root_id = btrfs_root_id(inode->root);
+++++++++++++ ++ ++++++++ const u64 ino = btrfs_ino(inode);
+++++++++++++ ++ ++++++++ u64 fsl;
+++++++++++++ ++ ++++++++ u64 cur;
+++++++++++++ ++ ++++++++ u64 end;
+++++++++++++ ++ ++++++++ bool ret = true;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!(btrfs_stripe_allow_rmw(fs_info) & which))
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, start);
+++++++++++++ ++ ++++++++ if (!bg)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ if (!btrfs_is_stripe_alloc_bg(bg)) {
+++++++++++++ ++ ++++++++ /* Not a raid56 data group: nothing forces COW here anyway. */
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ return true;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ fsl = bg->full_stripe_len;
+++++++++++++ ++ ++++++++ cur = bg->start + stripe_round_down(start - bg->start, fsl);
+++++++++++++ ++ ++++++++ end = start + num_bytes;
+++++++++++++ ++ ++++++++ while (cur < end) {
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ int verdict = 0; /* 0 = unknown, 1 = ok, -1 = foreign */
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (cur < run->start || cur >= run->end)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (run->class == BTRFS_STRIPE_RUN_NOCOW &&
+++++++++++++ ++ ++++++++ run->root == root_id && run->owner == ino)
+++++++++++++ ++ ++++++++ verdict = 1;
+++++++++++++ ++ ++++++++ else
+++++++++++++ ++ ++++++++ verdict = -1;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * No groups_sem here (this runs from the nocow check, not
+++++++++++++ ++ ++++++++ * from the allocator), so the reliable blocking search is
+++++++++++++ ++ ++++++++ * both safe and wanted: a spurious "not ours" would force
+++++++++++++ ++ ++++++++ * COW on an extent that does not need it. A NOWAIT write
+++++++++++++ ++ ++++++++ * must not block, so it gets the nonblocking search, and
+++++++++++++ ++ ++++++++ * COW (which it cannot do either) when that cannot answer.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (verdict == 0 &&
+++++++++++++ ++ ++++++++ !stripe_extents_owned_by(fs_info, root_id, ino, cur, fsl,
+++++++++++++ ++ ++++++++ NULL, nowait))
+++++++++++++ ++ ++++++++ verdict = -1;
+++++++++++++ ++ ++++++++ if (verdict < 0) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ cur += fsl;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Lower an allocation ceiling to the end of the last live sector at or above
+++++++++++++ ++ ++++++++ * @floor. Sectors between it and the frontier were allocated and abandoned:
+++++++++++++ ++ ++++++++ * they never get a bio, so a parked rbio counting on them would leave only
+++++++++++++ ++ ++++++++ * when kicked or at its park deadline. Caller holds bg->stripe_run_lock.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static u64 run_live_ceiling(const struct btrfs_open_stripe_run *run, u64 floor,
+++++++++++++ ++ ++++++++ u64 ceiling)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
+++++++++++++ ++ ++++++++ u32 first;
+++++++++++++ ++ ++++++++ u32 i;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!run->live || !run->live_nbits || floor < run->start ||
+++++++++++++ ++ ++++++++ ceiling <= floor)
+++++++++++++ ++ ++++++++ return ceiling;
+++++++++++++ ++ ++++++++ first = (floor - run->start) >> bits;
+++++++++++++ ++ ++++++++ i = (ceiling - run->start) >> bits;
+++++++++++++ ++ ++++++++ if (i > run->live_nbits)
+++++++++++++ ++ ++++++++ return ceiling;
+++++++++++++ ++ ++++++++ while (i > first && !test_bit(i - 1, run->live))
+++++++++++++ ++ ++++++++ i--;
+++++++++++++ ++ ++++++++ return run->start + ((u64)i << bits);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * How far has the covering run allocated into this stripe? Returns the
+++++++++++++ ++ ++++++++ * exclusive end of the allocated prefix, clamped to the stripe, or U64_MAX
+++++++++++++ ++ ++++++++ * when no run covers the stripe (nothing left to wait for; the caller
+++++++++++++ ++ ++++++++ * proceeds as if the stripe were fully allocated and fully arrived).
+++++++++++++ ++ ++++++++ * A partial write below the returned ceiling is waiting on neighbours whose
+++++++++++++ ++ ++++++++ * data IO is already in flight -- allocation happens at writeback submission
+++++++++++++ ++ ++++++++ * and the run's inflight accounting makes the commit wait for it -- so a
+++++++++++++ ++ ++++++++ * parked rbio covering [stripe_start, ceiling) can complete (padding the
+++++++++++++ ++ ++++++++ * dead tail), and one that does not should keep waiting for merges.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++u64 btrfs_stripe_run_alloc_ceiling(struct btrfs_fs_info *fs_info,
+++++++++++++ ++ ++++++++ u64 stripe_start, u64 stripe_len)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ u64 ceiling = U64_MAX;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++++ ++ ++++++++ return U64_MAX;
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+++++++++++++ ++ ++++++++ if (!bg)
+++++++++++++ ++ ++++++++ return U64_MAX;
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (stripe_start < run->start || stripe_start >= run->end)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ ceiling = clamp(run->offset, stripe_start,
+++++++++++++ ++ ++++++++ stripe_start + stripe_len);
+++++++++++++ ++ ++++++++ ceiling = run_live_ceiling(run, stripe_start, ceiling);
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ return ceiling;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Which sectors of this stripe may be padded?
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Fills @dead with one bit per sector, set where the sector holds nothing
+++++++++++++ ++ ++++++++ * anyone has written: never handed out, handed out and abandoned, or
+++++++++++++ ++ ++++++++ * handed out with the write not yet completed (per the run's written map).
+++++++++++++ ++ ++++++++ * The caller may zero-fill exactly those and write the whole stripe.
+++++++++++++ ++ ++++++++ * @unwritten, when non-NULL, gets the subset that is allocated but
+++++++++++++ ++ ++++++++ * unwritten.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Zeroing an allocated sector whose write is still to come is safe: the
+++++++++++++ ++ ++++++++ * caller holds the stripe lock, which every raid56 write to the stripe
+++++++++++++ ++ ++++++++ * takes, so that write lands after the caller's; completed writes are in
+++++++++++++ ++ ++++++++ * the written map before the stripe is unlocked (rbio_orig_end_io()); and
+++++++++++++ ++ ++++++++ * nothing reads the sector before its own write lands, since its extent is
+++++++++++++ ++ ++++++++ * not committed until then. A NOCOW run's written map is not consulted:
+++++++++++++ ++ ++++++++ * its live sectors are rewritten in place, so they may already hold data.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Returns false if no run covers the stripe; nothing may be padded then,
+++++++++++++ ++ ++++++++ * since an uncovered sector may hold committed data. @info, when non-NULL,
+++++++++++++ ++ ++++++++ * records the covering run's state for the pad-decline trace.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++bool btrfs_stripe_run_pad_mask(struct btrfs_fs_info *fs_info, u64 stripe_start,
+++++++++++++ ++ ++++++++ u64 stripe_len, unsigned long *dead,
+++++++++++++ ++ ++++++++ unsigned long *unwritten,
+++++++++++++ ++ ++++++++ struct btrfs_stripe_pad_info *info)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ const u32 bits = fs_info->sectorsize_bits;
+++++++++++++ ++ ++++++++ const u32 nsectors = stripe_len >> bits;
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ bool ret = false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (info) {
+++++++++++++ ++ ++++++++ memset(info, 0, sizeof(*info));
+++++++++++++ ++ ++++++++ info->reason = BTRFS_STRIPE_PAD_NO_RUN;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs)) {
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
+++++++++++++ ++ ++++++++ if (info)
+++++++++++++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_BGS;
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+++++++++++++ ++ ++++++++ if (!bg) {
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
+++++++++++++ ++ ++++++++ if (info)
+++++++++++++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_BG;
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (info)
+++++++++++++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_COVER;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ u32 first;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (stripe_start < run->start || stripe_start >= run->end)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (info) {
+++++++++++++ ++ ++++++++ info->run_start = run->start;
+++++++++++++ ++ ++++++++ info->run_end = run->end;
+++++++++++++ ++ ++++++++ info->run_offset = run->offset;
+++++++++++++ ++ ++++++++ info->inflight_bytes = run->inflight_bytes;
+++++++++++++ ++ ++++++++ info->run_open = run->open;
+++++++++++++ ++ ++++++++ info->run_class = run->class;
+++++++++++++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_LIVE;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (!run->live || !run->live_nbits)
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * The stripe must lie inside the run's span. A closed run
+++++++++++++ ++ ++++++++ * has shrunk its end to the frontier, so its final stripe
+++++++++++++ ++ ++++++++ * overhangs: the sectors past end are the returned tail,
+++++++++++++ ++ ++++++++ * sub-stripe free space nothing can claim while the stripe
+++++++++++++ ++ ++++++++ * holds live data. They were never allocated, their live
+++++++++++++ ++ ++++++++ * bits are clear, and they are exactly what a kicked park
+++++++++++++ ++ ++++++++ * needs to pad. An open run's end is stripe aligned, so
+++++++++++++ ++ ++++++++ * overhang there means a stripe the run does not own.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (stripe_start + stripe_len > run->end && run->open) {
+++++++++++++ ++ ++++++++ if (info)
+++++++++++++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_OVERHANG;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ first = (stripe_start - run->start) >> bits;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * And wholly inside the map. Declining here costs a
+++++++++++++ ++ ++++++++ * read-modify-write; reading past the map would be a lie about
+++++++++++++ ++ ++++++++ * which sectors are dead, and padding acts on that by
+++++++++++++ ++ ++++++++ * overwriting them.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (first + nsectors > run->live_nbits) {
+++++++++++++ ++ ++++++++ if (info)
+++++++++++++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_PAST_MAP;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ bitmap_zero(dead, nsectors);
+++++++++++++ ++ ++++++++ if (unwritten)
+++++++++++++ ++ ++++++++ bitmap_zero(unwritten, nsectors);
+++++++++++++ ++ ++++++++ for (u32 i = 0; i < nsectors; i++) {
+++++++++++++ ++ ++++++++ if (!test_bit(first + i, run->live)) {
+++++++++++++ ++ ++++++++ set_bit(i, dead);
+++++++++++++ ++ ++++++++ } else if (run->written &&
+++++++++++++ ++ ++++++++ run->class != BTRFS_STRIPE_RUN_NOCOW &&
+++++++++++++ ++ ++++++++ !test_bit(first + i, run->written)) {
+++++++++++++ ++ ++++++++ set_bit(i, dead);
+++++++++++++ ++ ++++++++ if (unwritten)
+++++++++++++ ++ ++++++++ set_bit(i, unwritten);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (info)
+++++++++++++ ++ ++++++++ info->reason = BTRFS_STRIPE_PAD_OK;
+++++++++++++ ++ ++++++++ ret = true;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (!ret)
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/* No open or draining run covers @bytenr any more. */
+++++++++++++ ++ ++++++++static bool stripe_log_range_settled(struct btrfs_block_group *bg, u64 bytenr)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ bool settled = true;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (bytenr >= run->start && bytenr < run->offset) {
+++++++++++++ ++ ++++++++ settled = false;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ return settled;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Remember, on the logging inode, a logged extent that lives in the final
+++++++++++++ ++ ++++++++ * partial stripe of one of the inode's recently closed runs, so that the
+++++++++++++ ++ ++++++++ * next fsync can re-dirty its file range and carry the data forward into
+++++++++++++ ++ ++++++++ * the then-current run -- letting the old partial stripe free whole. A
+++++++++++++ ++ ++++++++ * zero @file_len means the caller does not want this extent carried
+++++++++++++ ++ ++++++++ * (e.g. compressed extents, which would recompress). @ps_len is nonzero
+++++++++++++ ++ ++++++++ * when the settling walk could see the run and computed its final partial
+++++++++++++ ++ ++++++++ * stripe. Every table here is small and bounded; on overflow the extent
+++++++++++++ ++ ++++++++ * simply is not carried and its stripe frees the slow way. Caller holds
+++++++++++++ ++ ++++++++ * the inode's log_mutex, which serializes against the fsync-entry consumer.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static void log_carry_record(struct btrfs_inode *inode, u64 file_start,
+++++++++++++ ++ ++++++++ u64 file_len, u64 bytenr, u64 num_bytes,
+++++++++++++ ++ ++++++++ u64 ps_start, u64 ps_len)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_log_carry *carry = inode->log_carry;
+++++++++++++ ++ ++++++++ int n, i;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!carry) {
+++++++++++++ ++ ++++++++ carry = kzalloc(sizeof(*carry), GFP_NOFS);
+++++++++++++ ++ ++++++++ if (!carry)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ inode->log_carry = carry;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (ps_len) {
+++++++++++++ ++ ++++++++ n = min_t(int, carry->nr_ps, BTRFS_LOG_CARRY_NR_PS);
+++++++++++++ ++ ++++++++ for (i = 0; i < n; i++)
+++++++++++++ ++ ++++++++ if (carry->ps_start[i] == ps_start)
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ if (i == n) {
+++++++++++++ ++ ++++++++ int slot = carry->nr_ps % BTRFS_LOG_CARRY_NR_PS;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ carry->ps_start[slot] = ps_start;
+++++++++++++ ++ ++++++++ carry->ps_len[slot] = ps_len;
+++++++++++++ ++ ++++++++ carry->nr_ps++;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (!file_len || carry->nr_ranges >= BTRFS_LOG_CARRY_NR_RANGES)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ n = min_t(int, carry->nr_ps, BTRFS_LOG_CARRY_NR_PS);
+++++++++++++ ++ ++++++++ for (i = 0; i < n; i++)
+++++++++++++ ++ ++++++++ if (bytenr < carry->ps_start[i] + carry->ps_len[i] &&
+++++++++++++ ++ ++++++++ bytenr + num_bytes > carry->ps_start[i])
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ if (i == n)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ for (i = 0; i < carry->nr_ranges; i++)
+++++++++++++ ++ ++++++++ if (carry->range[i].start == file_start &&
+++++++++++++ ++ ++++++++ carry->range[i].len == file_len)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ carry->range[carry->nr_ranges].start = file_start;
+++++++++++++ ++ ++++++++ carry->range[carry->nr_ranges].len = file_len;
+++++++++++++ ++ ++++++++ carry->range[carry->nr_ranges].disk_bytenr = bytenr;
+++++++++++++ ++ ++++++++ carry->nr_ranges++;
+++++++++++++ ++ ++++++++ atomic64_inc(&inode->root->fs_info->stripe_run_stats.log_carry_ranges);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Disk end of @inode's last regular file extent, or 0: the re-adoption
+++++++++++++ ++ ++++++++ * candidate for a nocow allocation arriving with no allocation hint --
+++++++++++++ ++ ++++++++ * after a remount or eviction the extent maps are gone, so the normal
+++++++++++++ ++ ++++++++ * em-based hint is empty exactly when re-adoption matters most.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++u64 btrfs_nocow_alloc_hint(struct btrfs_inode *inode)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_root *root = inode->root;
+++++++++++++ ++ ++++++++ struct btrfs_path *path;
+++++++++++++ ++ ++++++++ struct btrfs_key key;
+++++++++++++ ++ ++++++++ struct btrfs_file_extent_item *fi;
+++++++++++++ ++ ++++++++ u64 hint = 0;
+++++++++++++ ++ ++++++++ int ret;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ path = btrfs_alloc_path();
+++++++++++++ ++ ++++++++ if (!path)
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++ key.objectid = btrfs_ino(inode);
+++++++++++++ ++ ++++++++ key.type = BTRFS_EXTENT_DATA_KEY;
+++++++++++++ ++ ++++++++ key.offset = (u64)-1;
+++++++++++++ ++ ++++++++ ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
+++++++++++++ ++ ++++++++ if (ret < 0)
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ ret = btrfs_previous_item(root, path, key.objectid,
+++++++++++++ ++ ++++++++ BTRFS_EXTENT_DATA_KEY);
+++++++++++++ ++ ++++++++ if (ret)
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
+++++++++++++ ++ ++++++++ fi = btrfs_item_ptr(path->nodes[0], path->slots[0],
+++++++++++++ ++ ++++++++ struct btrfs_file_extent_item);
+++++++++++++ ++ ++++++++ if (btrfs_file_extent_type(path->nodes[0], fi) ==
+++++++++++++ ++ ++++++++ BTRFS_FILE_EXTENT_INLINE)
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ if (btrfs_file_extent_disk_bytenr(path->nodes[0], fi))
+++++++++++++ ++ ++++++++ hint = btrfs_file_extent_disk_bytenr(path->nodes[0], fi) +
+++++++++++++ ++ ++++++++ btrfs_file_extent_disk_num_bytes(path->nodes[0], fi);
+++++++++++++ ++ ++++++++out:
+++++++++++++ ++ ++++++++ btrfs_free_path(path);
+++++++++++++ ++ ++++++++ return hint;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/* Does no run object (open or draining) overlap [start, start + len)? */
+++++++++++++ ++ ++++++++static bool stripe_range_run_free(struct btrfs_block_group *bg, u64 start,
+++++++++++++ ++ ++++++++ u64 len)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ bool ret = true;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (run->start < start + len && run->end > start) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/* Does any stripe run, open or draining, still cover part of @bg? */
+++++++++++++ ++ ++++++++bool btrfs_block_group_has_stripe_runs(struct btrfs_block_group *bg)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ bool ret;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ ret = !list_empty(&bg->open_stripe_runs);
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Close @inode's open LOG and NOCOW runs at eviction, so a persistent NOCOW
+++++++++++++ ++ ++++++++ * run's claimed tail does not stay unusable for the rest of the mount. The
+++++++++++++ ++ ++++++++ * tails return to the free space cache.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_close_inode_stripe_runs(struct btrfs_inode *inode)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = inode->root->fs_info;
+++++++++++++ ++ ++++++++ const u64 root_id = btrfs_root_id(inode->root);
+++++++++++++ ++ ++++++++ const u64 ino = btrfs_ino(inode);
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++restart:
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs,
+++++++++++++ ++ ++++++++ open_stripe_bg_list) {
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ u64 tail_start = 0;
+++++++++++++ ++ ++++++++ u64 tail_len = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (!run->open || run->root != root_id ||
+++++++++++++ ++ ++++++++ run->owner != ino ||
+++++++++++++ ++ ++++++++ (run->class != BTRFS_STRIPE_RUN_LOG &&
+++++++++++++ ++ ++++++++ run->class != BTRFS_STRIPE_RUN_NOCOW))
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (tail_len) {
+++++++++++++ ++ ++++++++ btrfs_get_block_group(bg);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ goto restart;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * A log commit on a shared LOG run: instead of closing the run, move its
+++++++++++++ ++ ++++++++ * frontier past the stripe holding the end of the logged extent, so nothing
+++++++++++++ ++ ++++++++ * allocates beside the extent after the log commit while the run goes on in
+++++++++++++ ++ ++++++++ * the next stripe. The skipped sectors are not live (the pad oracle fills
+++++++++++++ ++ ++++++++ * them) and go back to the free space cache like a closed run's tail: inside
+++++++++++++ ++ ++++++++ * a partly live stripe they count as trapped until the stripe frees, and a
+++++++++++++ ++ ++++++++ * later close returns only [offset, end), which no longer includes them.
+++++++++++++ ++ ++++++++ * Caller holds bg->stripe_run_lock. Returns the skipped length.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static u64 seal_log_stripe(struct btrfs_block_group *bg,
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run, u64 ext_end,
+++++++++++++ ++ ++++++++ u64 *skip_start)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++++ ++ ++++++++ u64 boundary;
+++++++++++++ ++ ++++++++ u64 skip;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!run->open)
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++ boundary = bg->start + stripe_round_down(ext_end - 1 - bg->start, fsl) + fsl;
+++++++++++++ ++ ++++++++ if (run->offset >= boundary)
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++ boundary = min(boundary, run->end);
+++++++++++++ ++ ++++++++ skip = boundary - run->offset;
+++++++++++++ ++ ++++++++ *skip_start = run->offset;
+++++++++++++ ++ ++++++++ run->offset = boundary;
+++++++++++++ ++ ++++++++ bg->stripe_open_remainder -= skip;
+++++++++++++ ++ ++++++++ if (run->offset == run->end) {
+++++++++++++ ++ ++++++++ int class;
+++++++++++++ ++ ++++++++ int band;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
+++++++++++++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+++++++++++++ ++ ++++++++ if (bg->open_stripe[class][band] == run)
+++++++++++++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++++ ++ ++++++++ run->open = false;
+++++++++++++ ++ ++++++++ if (open_stripe_run_drained(run) && !run->retained)
+++++++++++++ ++ ++++++++ free_open_stripe_run(bg, run);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ return skip;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++static bool run_unwritten_in(struct btrfs_open_stripe_run *run, u64 start,
+++++++++++++ ++ ++++++++ u64 len);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Per-stripe settle (stripe_log_data_shared): every allocated sector of the
+++++++++++++ ++ ++++++++ * stripes holding [bytenr, bytenr + num_bytes) has been written, or no run
+++++++++++++ ++ ++++++++ * covers them any more. Writers sharing those stripes park and merge into
+++++++++++++ ++ ++++++++ * the same full-stripe write, so this waits for them and for nothing else in
+++++++++++++ ++ ++++++++ * the run.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static bool stripe_log_extent_written(struct btrfs_block_group *bg,
+++++++++++++ ++ ++++++++ u64 bytenr, u64 num_bytes)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++++ ++ ++++++++ const u64 s = bg->start + stripe_round_down(bytenr - bg->start, fsl);
+++++++++++++ ++ ++++++++ const u64 e = bg->start +
+++++++++++++ ++ ++++++++ stripe_round_down(bytenr + num_bytes - bg->start + fsl - 1, fsl);
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ bool written = true;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (bytenr < run->start || bytenr >= run->offset)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (run_unwritten_in(run, s, e - s))
+++++++++++++ ++ ++++++++ written = false;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ return written;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * A log commit is about to make the extent at [bytenr, bytenr + num_bytes)
+++++++++++++ ++ ++++++++ * durable, after which no later write may extend or RMW its stripes. Close
+++++++++++++ ++ ++++++++ * the run covering the extent so nothing more allocates into its stripes,
+++++++++++++ ++ ++++++++ * kick any parked partial writes there, and wait for all IO into the run to
+++++++++++++ ++ ++++++++ * finish. This is a pure data-IO wait (no transaction joins), bounded by
+++++++++++++ ++ ++++++++ * bio flight time because parked writes are flushed first, and is safe
+++++++++++++ ++ ++++++++ * under the inode log mutex and a running transaction handle.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * With stripe_log_data_shared a shared LOG run is not closed: the stripe
+++++++++++++ ++ ++++++++ * holding the end of the extent is sealed (seal_log_stripe()). In that
+++++++++++++ ++ ++++++++ * mode the wait, for any run, is for every allocated sector of the
+++++++++++++ ++ ++++++++ * extent's stripes to be written (stripe_log_extent_written()) rather than
+++++++++++++ ++ ++++++++ * for the run to drain. Neither wait has a timeout: returning early would
+++++++++++++ ++ ++++++++ * let a neighbour's partial write RMW the extent after it is durable.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * [file_start, file_start + file_len) is the extent's file range, recorded
+++++++++++++ ++ ++++++++ * for carry-forward when the extent is in the run's final partial stripe
+++++++++++++ ++ ++++++++ * (stripe_log_data_shared clear only); file_len == 0 opts out of carrying.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_log_settle_stripes(struct btrfs_inode *inode, u64 file_start,
+++++++++++++ ++ ++++++++ u64 file_len, u64 bytenr, u64 num_bytes)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = inode->root->fs_info;
+++++++++++++ ++ ++++++++ /* Once: the seal and the wait below must agree on the mode. */
+++++++++++++ ++ ++++++++ const bool shared = READ_ONCE(fs_info->stripe_log_data_shared);
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ u64 tail_start = 0;
+++++++++++++ ++ ++++++++ u64 tail_len = 0;
+++++++++++++ ++ ++++++++ u64 flush_start = 0;
+++++++++++++ ++ ++++++++ u64 flush_len = 0;
+++++++++++++ ++ ++++++++ u64 ps_start = 0;
+++++++++++++ ++ ++++++++ u64 ps_len = 0;
+++++++++++++ ++ ++++++++ u64 seal_start = 0;
+++++++++++++ ++ ++++++++ u64 seal_len = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* stripe_alloc_allow_rmw=fsync waives the log-commit guarantee. */
+++++++++++++ ++ ++++++++ if (btrfs_stripe_allow_rmw(fs_info) & BTRFS_STRIPE_RMW_FSYNC)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
+++++++++++++ ++ ++++++++ if (!bg)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ u64 filled;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (bytenr < run->start || bytenr >= run->offset)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * A nodatacow file's logged extents get no fsync survival
+++++++++++++ ++ ++++++++ * guarantee -- the file's own later in-place writes can
+++++++++++++ ++ ++++++++ * always tear them -- so closing its run would trap the
+++++++++++++ ++ ++++++++ * tail for nothing.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (run->class == BTRFS_STRIPE_RUN_NOCOW)
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ if (shared && (run->owner & BIT_ULL(63))) {
+++++++++++++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* only the logged extent's stripes, not the shared run */
+++++++++++++ ++ ++++++++ flush_start = bg->start + stripe_round_down(bytenr - bg->start, fsl);
+++++++++++++ ++ ++++++++ flush_len = bg->start + stripe_round_down(bytenr + num_bytes - 1 - bg->start, fsl) +
+++++++++++++ ++ ++++++++ fsl - flush_start;
+++++++++++++ ++ ++++++++ seal_len = seal_log_stripe(bg, run, bytenr + num_bytes, &seal_start);
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ flush_start = run->start;
+++++++++++++ ++ ++++++++ flush_len = run->end - run->start;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * The run's final partial stripe, whether this call closes
+++++++++++++ ++ ++++++++ * the run or an earlier settle already did (offset froze at
+++++++++++++ ++ ++++++++ * close). Runs start stripe-aligned, so alignment relative
+++++++++++++ ++ ++++++++ * to run->start is stripe alignment.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ filled = run->offset - run->start;
+++++++++++++ ++ ++++++++ if (!stripe_is_aligned(filled, bg->full_stripe_len)) {
+++++++++++++ ++ ++++++++ ps_start = run->start +
+++++++++++++ ++ ++++++++ stripe_round_down(filled, bg->full_stripe_len);
+++++++++++++ ++ ++++++++ ps_len = bg->full_stripe_len;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (run->open) {
+++++++++++++ ++ ++++++++ int class;
+++++++++++++ ++ ++++++++ int band;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
+++++++++++++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+++++++++++++ ++ ++++++++ if (bg->open_stripe[class][band] == run)
+++++++++++++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!shared)
+++++++++++++ ++ ++++++++ log_carry_record(inode, file_start, file_len, bytenr, num_bytes,
+++++++++++++ ++ ++++++++ ps_start, ps_len);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (seal_len) {
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, seal_start, seal_len);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.log_seals);
+++++++++++++ ++ ++++++++ atomic64_add(seal_len, &fs_info->stripe_run_stats.log_seal_bytes);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (tail_len) {
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (flush_len) {
+++++++++++++ ++ ++++++++ ktime_t t0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, flush_start, flush_len, false);
+++++++++++++ ++ ++++++++ t0 = ktime_get();
+++++++++++++ ++ ++++++++ if (shared)
+++++++++++++ ++ ++++++++ wait_var_event(&bg->open_stripe_runs,
+++++++++++++ ++ ++++++++ stripe_log_extent_written(bg, bytenr, num_bytes) ||
+++++++++++++ ++ ++++++++ BTRFS_FS_ERROR(fs_info));
+++++++++++++ ++ ++++++++ else
+++++++++++++ ++ ++++++++ wait_var_event(&bg->open_stripe_runs,
+++++++++++++ ++ ++++++++ stripe_log_range_settled(bg, bytenr) ||
+++++++++++++ ++ ++++++++ BTRFS_FS_ERROR(fs_info));
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.log_settle_waits);
+++++++++++++ ++ ++++++++ atomic64_add(ktime_to_ns(ktime_sub(ktime_get(), t0)),
+++++++++++++ ++ ++++++++ &fs_info->stripe_run_stats.log_settle_wait_ns);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Can stripe-exclusive allocation be enabled on this filesystem? Called for
+++++++++++++ ++ ++++++++ * the "stripe_alloc" filesystem property, on a mounted (or mounting, options
+++++++++++++ ++ ++++++++ * finalized) filesystem.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++int btrfs_stripe_alloc_check_support(struct btrfs_fs_info *fs_info)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ if (!btrfs_test_opt(fs_info, FREE_SPACE_TREE)) {
+++++++++++++ ++ ++++++++ btrfs_err(fs_info,
+++++++++++++ ++ ++++++++ "stripe_alloc requires the free space tree (space_cache=v2)");
+++++++++++++ ++ ++++++++ return -EINVAL;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (btrfs_fs_incompat(fs_info, REMAP_TREE)) {
+++++++++++++ ++ ++++++++ btrfs_err(fs_info,
+++++++++++++ ++ ++++++++ "stripe_alloc is not supported together with the remap-tree feature");
+++++++++++++ ++ ++++++++ return -EINVAL;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (btrfs_is_zoned(fs_info)) {
+++++++++++++ ++ ++++++++ btrfs_err(fs_info,
+++++++++++++ ++ ++++++++ "stripe_alloc is not supported on zoned filesystems");
+++++++++++++ ++ ++++++++ return -EINVAL;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * A mixed group is excluded from the claim rule anyway
+++++++++++++ ++ ++++++++ * (btrfs_is_stripe_alloc_bg() requires DATA without METADATA), so
+++++++++++++ ++ ++++++++ * enabling here would protect nothing while reporting success.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (btrfs_fs_incompat(fs_info, MIXED_GROUPS)) {
+++++++++++++ ++ ++++++++ btrfs_err(fs_info,
+++++++++++++ ++ ++++++++ "stripe_alloc is not supported on mixed block groups");
+++++++++++++ ++ ++++++++ return -EINVAL;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Arm or disarm stripe_unusable accounting across all raid56 data and
+++++++++++++ ++ ++++++++ * metadata block groups whose caches are loaded. Same walk as the
+++++++++++++ ++ ++++++++ * commit-time rescan.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static void stripe_alloc_sweep_groups(struct btrfs_fs_info *fs_info, bool arm)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_space_info *sinfo;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ list_for_each_entry(sinfo, &fs_info->space_info, list) {
+++++++++++++ ++ ++++++++ int raid;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!(sinfo->flags & (BTRFS_BLOCK_GROUP_DATA |
+++++++++++++ ++ ++++++++ BTRFS_BLOCK_GROUP_METADATA |
+++++++++++++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Metadata: the scan that maintains the stripe_meta margin
+++++++++++++ ++ ++++++++ * and reserve stops with the option, so clear them here or
+++++++++++++ ++ ++++++++ * they hold metadata back forever. The groups themselves
+++++++++++++ ++ ++++++++ * are armed and disarmed below like data groups: raid56
+++++++++++++ ++ ++++++++ * metadata places tree blocks in whole stripes from the
+++++++++++++ ++ ++++++++ * moment the option is on, so its trapped and claimable
+++++++++++++ ++ ++++++++ * bytes count from that moment and stop counting when it
+++++++++++++ ++ ++++++++ * goes off.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (!arm && (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA)) {
+++++++++++++ ++ ++++++++ spin_lock(&sinfo->lock);
+++++++++++++ ++ ++++++++ sinfo->bytes_stripe_margin = 0;
+++++++++++++ ++ ++++++++ sinfo->bytes_stripe_reserve = 0;
+++++++++++++ ++ ++++++++ btrfs_try_granting_tickets(sinfo);
+++++++++++++ ++ ++++++++ spin_unlock(&sinfo->lock);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ /* The writeback quarantine is the gate's; it goes with it. */
+++++++++++++ ++ ++++++++ if (!arm && (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)) {
+++++++++++++ ++ ++++++++ spin_lock(&sinfo->lock);
+++++++++++++ ++ ++++++++ sinfo->bytes_stripe_quarantine = 0;
+++++++++++++ ++ ++++++++ btrfs_try_granting_tickets(sinfo);
+++++++++++++ ++ ++++++++ spin_unlock(&sinfo->lock);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ down_read(&sinfo->groups_sem);
+++++++++++++ ++ ++++++++ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!(btrfs_raid_array[raid].bg_flag &
+++++++++++++ ++ ++++++++ BTRFS_BLOCK_GROUP_RAID56_MASK))
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ list_for_each_entry(bg, &sinfo->block_groups[raid],
+++++++++++++ ++ ++++++++ list) {
+++++++++++++ ++ ++++++++ if (!arm)
+++++++++++++ ++ ++++++++ btrfs_block_group_disarm_stripe_unusable(bg);
+++++++++++++ ++ ++++++++ else if (bg->cached == BTRFS_CACHE_FINISHED)
+++++++++++++ ++ ++++++++ btrfs_block_group_init_stripe_unusable(bg);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ up_read(&sinfo->groups_sem);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Drain the writes the legacy allocator placed before stripe_alloc was
+++++++++++++ ++ ++++++++ * enabled on a live filesystem, then arm the write-hole invariant check.
+++++++++++++ ++ ++++++++ * Runs in a worker: the property that enables the policy is applied inside
+++++++++++++ ++ ++++++++ * a transaction, where flushing delalloc cannot wait.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_stripe_alloc_enable_work(struct work_struct *work)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = container_of(work, struct btrfs_fs_info,
+++++++++++++ ++ ++++++++ stripe_alloc_enable_work);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ btrfs_start_delalloc_roots(fs_info, LONG_MAX, false);
+++++++++++++ ++ ++++++++ btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Tree blocks the legacy allocator placed in the transaction running at
+++++++++++++ ++ ++++++++ * the flip are written into partly used stripes by that transaction's
+++++++++++++ ++ ++++++++ * commit. Wait for it before arming the check, which would otherwise
+++++++++++++ ++ ++++++++ * report those writes as violations.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ btrfs_commit_current_transaction(fs_info->tree_root);
+++++++++++++ ++ ++++++++ clear_bit(BTRFS_FS_STRIPE_ALLOC_ENABLING, &fs_info->flags);
+++++++++++++ ++ ++++++++ btrfs_info(fs_info,
+++++++++++++ ++ ++++++++ "stripe_alloc: writes placed before the policy was enabled have landed");
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Turn on stripe-exclusive allocation, from the "stripe_alloc" filesystem
+++++++++++++ ++ ++++++++ * property on the top-level subvolume's root directory. Runs during mount
+++++++++++++ ++ ++++++++ * when the root directory inode loads its properties -- before any user IO
+++++++++++++ ++ ++++++++ * -- or at runtime from btrfs_set_prop().
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++int btrfs_enable_stripe_alloc(struct btrfs_fs_info *fs_info)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ bool live = test_bit(BTRFS_FS_OPEN, &fs_info->flags);
+++++++++++++ ++ ++++++++ int ret;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++ ret = btrfs_stripe_alloc_check_support(fs_info);
+++++++++++++ ++ ++++++++ if (ret)
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * On a live filesystem the legacy allocator may still have writes in
+++++++++++++ ++ ++++++++ * flight. Their extents sit in partly used stripes that no stripe run
+++++++++++++ ++ ++++++++ * covers, so the write-hole invariant check would report them although
+++++++++++++ ++ ++++++++ * they were placed before the policy existed. Keep the check off until
+++++++++++++ ++ ++++++++ * every ordered extent that could have been allocated before the flip
+++++++++++++ ++ ++++++++ * has landed; extents allocated after it have stripe runs. A mount-time
+++++++++++++ ++ ++++++++ * enable has nothing in flight and skips the drain.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (live)
+++++++++++++ ++ ++++++++ set_bit(BTRFS_FS_STRIPE_ALLOC_ENABLING, &fs_info->flags);
+++++++++++++ ++ ++++++++ btrfs_set_opt(fs_info->mount_opt, STRIPE_ALLOC);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Arm stripe_unusable accounting for raid56 data groups whose caches
+++++++++++++ ++ ++++++++ * loaded before the policy was enabled: block groups are read before
+++++++++++++ ++ ++++++++ * the root directory's properties apply during mount, and a runtime
+++++++++++++ ++ ++++++++ * property set can arrive at any point. Groups still caching arm
+++++++++++++ ++ ++++++++ * when their caching thread finishes, now that the policy is on.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ stripe_alloc_sweep_groups(fs_info, true);
+++++++++++++ ++ ++++++++ /* At mount, btrfs_fill_super() reports the outcome. */
+++++++++++++ ++ ++++++++ if (live) {
+++++++++++++ ++ ++++++++ btrfs_info(fs_info,
+++++++++++++ ++ ++++++++ "using stripe-exclusive allocation for raid56 data (filesystem property)");
+++++++++++++ ++ ++++++++ queue_work(system_unbound_wq, &fs_info->stripe_alloc_enable_work);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ return 0;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Turn off stripe-exclusive allocation (property deleted). Open runs are
+++++++++++++ ++ ++++++++ * not touched here: the commit-time retirement runs unconditionally and
+++++++++++++ ++ ++++++++ * drains them at the next transaction commit.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_disable_stripe_alloc(struct btrfs_fs_info *fs_info)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ /* Disarm while the policy bit still passes the per-group gate. */
+++++++++++++ ++ ++++++++ stripe_alloc_sweep_groups(fs_info, false);
+++++++++++++ ++ ++++++++ btrfs_clear_opt(fs_info->mount_opt, STRIPE_ALLOC);
+++++++++++++ ++ ++++++++ btrfs_info(fs_info,
+++++++++++++ ++ ++++++++ "stripe-exclusive allocation for raid56 data disabled");
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Does @logical lie within a stripe run, open or draining? The raid56 layer
+++++++++++++ ++ ++++++++ * uses this to decide whether a partial write may be parked to merge into a
+++++++++++++ ++ ++++++++ * full-stripe write. An open run fills the rest of the stripe later or
+++++++++++++ ++ ++++++++ * never; a draining run's frontier is frozen, so everything below it is IO
+++++++++++++ ++ ++++++++ * the commit drain already waits for and everything above it is dead space
+++++++++++++ ++ ++++++++ * padding may fill.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++bool btrfs_stripe_in_open_run(struct btrfs_fs_info *fs_info, u64 logical)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ return btrfs_stripe_open_run_class(fs_info, logical, NULL);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * The part of [*start, *start + *len) that @run owns and tracks in its
+++++++++++++ ++ ++++++++ * written map: inside [run->start, run->end) -- the map itself reaches
+++++++++++++ ++ ++++++++ * further (it spans at least BTRFS_STRIPE_RUN_LIVE_STRIPES stripes from
+++++++++++++ ++ ++++++++ * run->start, past run->end into later runs' ranges), so it must never be
+++++++++++++ ++ ++++++++ * consulted outside the run's own range -- and inside the map. Nodatacow
+++++++++++++ ++ ++++++++ * runs keep no written map: they are rewritten in place for as long as the
+++++++++++++ ++ ++++++++ * inode lives and are never complete. Caller holds stripe_run_lock.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static bool run_written_range(const struct btrfs_open_stripe_run *run,
+++++++++++++ ++ ++++++++ u64 *start, u64 *len)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
+++++++++++++ ++ ++++++++ u64 lo, hi;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (run->class == BTRFS_STRIPE_RUN_NOCOW)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ if (!run->live || !run->written || !run->live_nbits)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ lo = max(*start, run->start);
+++++++++++++ ++ ++++++++ hi = min3(*start + *len, run->end,
+++++++++++++ ++ ++++++++ run->start + ((u64)run->live_nbits << bits));
+++++++++++++ ++ ++++++++ if (lo >= hi)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ *start = lo;
+++++++++++++ ++ ++++++++ *len = hi - lo;
+++++++++++++ ++ ++++++++ return true;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * A raid56 write of [start, start + len) has completed: mark it written in
+++++++++++++ ++ ++++++++ * the run that owns it. Sectors that are not live in that run (freed while
+++++++++++++ ++ ++++++++ * the write was in flight, or a bookkeeping bug) are counted in
+++++++++++++ ++ ++++++++ * written_not_live: the pad oracle trusts this map to say which allocated
+++++++++++++ ++ ++++++++ * sectors hold data, so a write it misses could be zeroed.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_stripe_run_mark_written(struct btrfs_fs_info *fs_info, u64 start,
+++++++++++++ ++ ++++++++ u64 len)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ const u32 bits = fs_info->sectorsize_bits;
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ u32 not_live = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, start);
+++++++++++++ ++ ++++++++ if (!bg)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ u64 s = start, l = len;
+++++++++++++ ++ ++++++++ u32 first, nr, i;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!run_written_range(run, &s, &l))
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ first = (s - run->start) >> bits;
+++++++++++++ ++ ++++++++ nr = l >> bits;
+++++++++++++ ++ ++++++++ for (i = first; i < first + nr; i++)
+++++++++++++ ++ ++++++++ if (!test_bit(i, run->live))
+++++++++++++ ++ ++++++++ not_live++;
+++++++++++++ ++ ++++++++ bitmap_set(run->written, first, nr);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ /* Per-stripe log settles wait on the written map, not on a drain. */
+++++++++++++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ if (not_live)
+++++++++++++ ++ ++++++++ atomic64_add(not_live, &fs_info->stripe_park_stats.written_not_live);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/* Has @run allocated sectors in [start, start + len) it has not yet written? */
+++++++++++++ ++ ++++++++static bool run_unwritten_in(struct btrfs_open_stripe_run *run, u64 start,
+++++++++++++ ++ ++++++++ u64 len)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
+++++++++++++ ++ ++++++++ u32 first, last, i;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!run_written_range(run, &start, &len))
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ first = (start - run->start) >> bits;
+++++++++++++ ++ ++++++++ last = first + (len >> bits);
+++++++++++++ ++ ++++++++ for (i = first; i < last; i++)
+++++++++++++ ++ ++++++++ if (test_bit(i, run->live) && !test_bit(i, run->written))
+++++++++++++ ++ ++++++++ return true;
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * As btrfs_stripe_run_incomplete(), returning the class of the run that
+++++++++++++ ++ ++++++++ * leaves the stripe incomplete, or -1 if none does.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++int btrfs_stripe_run_incomplete_class(struct btrfs_fs_info *fs_info,
+++++++++++++ ++ ++++++++ u64 stripe_start, u64 stripe_len)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ int ret = -1;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++++ ++ ++++++++ return -1;
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+++++++++++++ ++ ++++++++ if (!bg)
+++++++++++++ ++ ++++++++ return -1;
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if ((run->open && run->offset > stripe_start &&
+++++++++++++ ++ ++++++++ run->offset < stripe_start + stripe_len) ||
+++++++++++++ ++ ++++++++ run_unwritten_in(run, stripe_start, stripe_len)) {
+++++++++++++ ++ ++++++++ ret = run->class;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Is [stripe_start, stripe_start + stripe_len) still waiting for writes from
+++++++++++++ ++ ++++++++ * a stripe run? Two ways:
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * - an open run's allocation frontier lies strictly inside it: the run will
+++++++++++++ ++ ++++++++ * allocate and write the rest of the stripe later, although every sector
+++++++++++++ ++ ++++++++ * allocated so far may already be written;
+++++++++++++ ++ ++++++++ * - a run covering it has sectors allocated (live) but not yet written: the
+++++++++++++ ++ ++++++++ * frontier has moved past, and the writes are still in flight (writeback
+++++++++++++ ++ ++++++++ * allocates a whole range before any of its bios complete).
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Either way the stripe has been written in part, and the rest will arrive
+++++++++++++ ++ ++++++++ * as a sub-stripe write that needs the part already written.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++bool btrfs_stripe_run_incomplete(struct btrfs_fs_info *fs_info,
+++++++++++++ ++ ++++++++ u64 stripe_start, u64 stripe_len)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ return btrfs_stripe_run_incomplete_class(fs_info, stripe_start,
+++++++++++++ ++ ++++++++ stripe_len) >= 0;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/* Open stripe runs by class, for sysfs: @counts has NR_CLASSES entries. */
+++++++++++++ ++ ++++++++void btrfs_count_open_stripe_runs(struct btrfs_fs_info *fs_info, int *counts)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ memset(counts, 0, sizeof(*counts) * BTRFS_STRIPE_RUN_NR_CLASSES);
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list) {
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list)
+++++++++++++ ++ ++++++++ if (run->open && run->class < BTRFS_STRIPE_RUN_NR_CLASSES)
+++++++++++++ ++ ++++++++ counts[run->class]++;
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Like btrfs_stripe_in_open_run(), also reporting the covering run's
+++++++++++++ ++ ++++++++ * class: the raid56 parking gate treats NOCOW-class runs specially
+++++++++++++ ++ ++++++++ * (sync writes into them never park; see rbio_try_park()).
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++bool btrfs_stripe_open_run_class(struct btrfs_fs_info *fs_info, u64 logical,
+++++++++++++ ++ ++++++++ enum btrfs_stripe_run_class *class)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ bool ret = false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, logical);
+++++++++++++ ++ ++++++++ if (!bg)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (logical < run->start || logical >= run->end)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Draining runs match too (see btrfs_stripe_in_open_run()):
+++++++++++++ ++ ++++++++ * a closed run's end is shrunk to its frontier, so a write
+++++++++++++ ++ ++++++++ * into it is below the frontier and its stripes are still
+++++++++++++ ++ ++++++++ * exclusive.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (class)
+++++++++++++ ++ ++++++++ *class = run->class;
+++++++++++++ ++ ++++++++ ret = true;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Report bytes that will never be written: an allocation released before
+++++++++++++ ++ ++++++++ * its write was issued. Same accounting as a completion, plus the sectors
+++++++++++++ ++ ++++++++ * go dead so a later partial write to their stripe can pad over them
+++++++++++++ ++ ++++++++ * instead of falling back to read-modify-write.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_open_stripe_write_abandoned(struct btrfs_block_group *bg, u64 start,
+++++++++++++ ++ ++++++++ u64 num_bytes)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (start >= run->start && start < run->offset) {
+++++++++++++ ++ ++++++++ ASSERT(start + num_bytes <= run->offset);
+++++++++++++ ++ ++++++++ ASSERT(run->inflight_bytes >= num_bytes);
+++++++++++++ ++ ++++++++ run_live_set(run, start, num_bytes, false);
+++++++++++++ ++ ++++++++ run->inflight_bytes -= num_bytes;
+++++++++++++ ++ ++++++++ open_stripe_run_drain_reported(bg, run);
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Report completed IO for an allocation from any stripe run. Every
+++++++++++++ ++ ++++++++ * allocated byte must be reported exactly once, here or through
+++++++++++++ ++ ++++++++ * btrfs_open_stripe_write_abandoned(), in any number of pieces (e.g. after
+++++++++++++ ++ ++++++++ * ordered extent splits).
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_open_stripe_write_done(struct btrfs_block_group *bg, u64 start,
+++++++++++++ ++ ++++++++ u64 num_bytes)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (start >= run->start && start < run->offset) {
+++++++++++++ ++ ++++++++ ASSERT(start + num_bytes <= run->offset);
+++++++++++++ ++ ++++++++ ASSERT(run->inflight_bytes >= num_bytes);
+++++++++++++ ++ ++++++++ run->inflight_bytes -= num_bytes;
+++++++++++++ ++ ++++++++ open_stripe_run_drain_reported(bg, run);
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Range variant of btrfs_open_stripe_write_done() for callers without the
+++++++++++++ ++ ++++++++ * block group at hand (e.g. the prealloc path, which never issues data IO
+++++++++++++ ++ ++++++++ * for its extent and reports it done at insertion).
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_open_stripe_write_done_bytenr(struct btrfs_fs_info *fs_info,
+++++++++++++ ++ ++++++++ u64 start, u64 num_bytes)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, start);
+++++++++++++ ++ ++++++++ if (!bg)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ btrfs_open_stripe_write_done(bg, start, num_bytes);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Find the open stripe run whose allocated region contains
+++++++++++++ ++ ++++++++ * [bytenr, bytenr + num_bytes). The caller must own unreported inflight
+++++++++++++ ++ ++++++++ * bytes in that region (i.e. it was allocated from the run and not yet
+++++++++++++ ++ ++++++++ * reported done), which is what keeps the returned pointer valid: a run
+++++++++++++ ++ ++++++++ * with inflight bytes is never freed.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++struct btrfs_open_stripe_run *btrfs_get_open_stripe_run(
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info, u64 bytenr, u64 num_bytes,
+++++++++++++ ++ ++++++++ u64 *open_seq)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run = NULL;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *iter;
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
+++++++++++++ ++ ++++++++ if (!bg)
+++++++++++++ ++ ++++++++ return NULL;
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(iter, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (bytenr >= iter->start && bytenr < iter->offset) {
+++++++++++++ ++ ++++++++ ASSERT(bytenr + num_bytes <= iter->offset);
+++++++++++++ ++ ++++++++ ASSERT(iter->inflight_bytes >= num_bytes);
+++++++++++++ ++ ++++++++ run = iter;
+++++++++++++ ++ ++++++++ *open_seq = iter->open_seq;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ return run;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Pointer-based variant of btrfs_open_stripe_write_done() for callers that
+++++++++++++ ++ ++++++++ * recorded the run at allocation or ordered extent creation time. The run
+++++++++++++ ++ ++++++++ * pointer stays valid until the reporter's own bytes are reported: a run
+++++++++++++ ++ ++++++++ * with inflight bytes is never freed.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * An ordered extent's data IO is over, whether or not all of it reached the
+++++++++++++ ++ ++++++++ * disk: stop counting it as in flight, and mark its sectors written. A
+++++++++++++ ++ ++++++++ * sector whose write failed or was never issued (truncation, a short direct
+++++++++++++ ++ ++++++++ * write, error teardown) will not be written now. Marking it keeps a log
+++++++++++++ ++ ++++++++ * settle from waiting for it forever, and a written sector is never padded,
+++++++++++++ ++ ++++++++ * so a later partial write to the stripe reads it instead of zeroing it.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Returns the number of allocated sectors that had not been written, and
+++++++++++++ ++ ++++++++ * in @first_unwritten the logical address of the lowest of them (U64_MAX if
+++++++++++++ ++ ++++++++ * there are none).
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++u32 btrfs_open_stripe_write_done_run(struct btrfs_open_stripe_run *run,
+++++++++++++ ++ ++++++++ u64 start, u64 num_bytes,
+++++++++++++ ++ ++++++++ u64 *first_unwritten)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg = run->bg;
+++++++++++++ ++ ++++++++ const u32 bits = bg->fs_info->sectorsize_bits;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ u64 s = start;
+++++++++++++ ++ ++++++++ u64 l = num_bytes;
+++++++++++++ ++ ++++++++ u32 unwritten = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ *first_unwritten = U64_MAX;
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ ASSERT(run->inflight_bytes >= num_bytes);
+++++++++++++ ++ ++++++++ run->inflight_bytes -= num_bytes;
+++++++++++++ ++ ++++++++ if (run_written_range(run, &s, &l)) {
+++++++++++++ ++ ++++++++ const u32 first = (s - run->start) >> bits;
+++++++++++++ ++ ++++++++ const u32 last = first + (l >> bits);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ for (u32 i = first; i < last; i++) {
+++++++++++++ ++ ++++++++ if (!test_bit(i, run->live) || test_bit(i, run->written))
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (!unwritten)
+++++++++++++ ++ ++++++++ *first_unwritten = run->start + ((u64)i << bits);
+++++++++++++ ++ ++++++++ unwritten++;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ bitmap_set(run->written, first, l >> bits);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ /* Under the lock: the run and the group may be gone after it. */
+++++++++++++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
+++++++++++++ ++ ++++++++ open_stripe_run_drain_reported(bg, run);
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ return unwritten;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Close the block group's open runs that predate @seq and return their
+++++++++++++ ++ ++++++++ * unallocated tails to the free space cache. Does not wait for inflight
+++++++++++++ ++ ++++++++ * IO. Metadata runs are left to btrfs_retire_meta_stripes() and persistent
+++++++++++++ ++ ++++++++ * NOCOW runs stay open unless dead, except when @seq is U64_MAX (read-only,
+++++++++++++ ++ ++++++++ * removal, unmount), which closes everything.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static void close_block_group_stripe_runs(struct btrfs_block_group *bg,
+++++++++++++ ++ ++++++++ u64 seq)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ int class, band;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++restart:
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ u64 tail_start;
+++++++++++++ ++ ++++++++ u64 tail_len;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!run->open || run->open_seq >= seq)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * A persistent NOCOW run holds only its inode's
+++++++++++++ ++ ++++++++ * write-hole-waived data, so commit retirement protects
+++++++++++++ ++ ++++++++ * nothing there, and closing it each commit would waste a
+++++++++++++ ++ ++++++++ * stripe per commit on a slowly appended file. It closes at
+++++++++++++ ++ ++++++++ * forced quiescing (seq == U64_MAX), at the inode's eviction,
+++++++++++++ ++ ++++++++ * or once it is dead.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (run->persistent && seq != U64_MAX) {
+++++++++++++ ++ ++++++++ /* A dead run protects nothing, only strands stripes. */
+++++++++++++ ++ ++++++++ if (!open_stripe_run_dead(run))
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.nocow_run_dead_closed);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Metadata runs are still being filled at this point in the
+++++++++++++ ++ ++++++++ * commit and their tree blocks are not written until later;
+++++++++++++ ++ ++++++++ * btrfs_retire_meta_stripes() closes them once they are.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class) && seq != U64_MAX)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
+++++++++++++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+++++++++++++ ++ ++++++++ if (bg->open_stripe[class][band] == run)
+++++++++++++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Closing may have freed the run, and returning its tail to
+++++++++++++ ++ ++++++++ * the free space cache needs the lock dropped, so rescan.
+++++++++++++ ++ ++++++++ * Each pass closes one run and open runs are bounded (band
+++++++++++++ ++ ++++++++ * slots plus the private runs of inodes logged this window).
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (tail_len)
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ goto restart;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Close all of a block group's open stripe runs without waiting for their
+++++++++++++ ++ ++++++++ * inflight IO to drain.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_close_bg_open_stripes(struct btrfs_block_group *bg)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ close_block_group_stripe_runs(bg, U64_MAX);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/* No run opened before @seq remains (open or draining). */
+++++++++++++ ++ ++++++++static bool bg_open_stripes_settled(struct btrfs_block_group *bg, u64 seq)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ bool ret = true;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ /* Persistent nocow runs are not part of any commit window. */
+++++++++++++ ++ ++++++++ if (run->persistent && seq != U64_MAX)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /* Nor are retained relocation runs, once nothing is in flight. */
+++++++++++++ ++ ++++++++ if (run->retained && open_stripe_run_drained(run) &&
+++++++++++++ ++ ++++++++ seq != U64_MAX)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /* Metadata drains after tree writeback, not here. */
+++++++++++++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class) && seq != U64_MAX)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (run->open_seq < seq) {
+++++++++++++ ++ ++++++++ ret = false;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Stop retaining @bg's relocation runs: free the ones already drained, and
+++++++++++++ ++ ++++++++ * let any still open or in flight be freed when they drain, like any other.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static void release_retained_stripe_runs(struct btrfs_block_group *bg)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run, *tmp;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry_safe(run, tmp, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (!run->retained)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ run->retained = false;
+++++++++++++ ++ ++++++++ atomic64_inc(&bg->fs_info->stripe_run_stats.reloc_run_released);
+++++++++++++ ++ ++++++++ if (open_stripe_run_drained(run))
+++++++++++++ ++ ++++++++ free_open_stripe_run(bg, run);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Relocation is done with its destination stripes: release its retained
+++++++++++++ ++ ++++++++ * runs in every block group (the dedication of btrfs_release_data_reloc_bg()
+++++++++++++ ++ ++++++++ * can have moved between groups during the relocation).
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static void btrfs_release_stripe_reloc_runs(struct btrfs_fs_info *fs_info)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
+++++++++++++ ++ ++++++++ release_retained_stripe_runs(bg);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/* Does @bg have metadata stripe runs? */
+++++++++++++ ++ ++++++++static bool bg_has_meta_runs(struct btrfs_block_group *bg)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ bool ret = false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class)) {
+++++++++++++ ++ ++++++++ ret = true;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Close and drain all of a block group's stripe runs. The caller must
+++++++++++++ ++ ++++++++ * prevent new allocations first (e.g. the block group is read-only),
+++++++++++++ ++ ++++++++ * otherwise this can wait forever. Membership in fs_info->open_stripe_bgs
+++++++++++++ ++ ++++++++ * is left for the next retire walk to collect.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * The wait also ends when the filesystem is in error: after a transaction
+++++++++++++ ++ ++++++++ * abort the ordered extents holding a run's bytes may never complete. The
+++++++++++++ ++ ++++++++ * runs are then left for btrfs_free_open_stripe_runs() at teardown.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_retire_block_group_stripes(struct btrfs_block_group *bg)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++++ ++ ++++++++ bool warned = false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Nothing may outlive this, relocation's retained runs included. */
+++++++++++++ ++ ++++++++ release_retained_stripe_runs(bg);
+++++++++++++ ++ ++++++++ close_block_group_stripe_runs(bg, U64_MAX);
+++++++++++++ ++ ++++++++ /* Parked partial-stripe rbios hold the very bios the wait drains. */
+++++++++++++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, bg->start, bg->length, false);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Metadata runs drain when their tree blocks are written and are
+++++++++++++ ++ ++++++++ * freed by btrfs_retire_meta_stripes(), both in a commit. Commit now
+++++++++++++ ++ ++++++++ * instead of waiting for the next one; no caller holds a transaction
+++++++++++++ ++ ++++++++ * handle.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (bg_has_meta_runs(bg))
+++++++++++++ ++ ++++++++ btrfs_commit_current_transaction(fs_info->tree_root);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * No timeout: a run that does not drain has an allocation that was
+++++++++++++ ++ ++++++++ * never reported written or abandoned, an accounting bug to fix, not
+++++++++++++ ++ ++++++++ * to wait out. Say which runs once, and keep waiting.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ while (!wait_var_event_timeout(&bg->open_stripe_runs,
+++++++++++++ ++ ++++++++ bg_open_stripes_settled(bg, U64_MAX) ||
+++++++++++++ ++ ++++++++ BTRFS_FS_ERROR(fs_info), 60 * HZ)) {
+++++++++++++ ++ ++++++++ if (warned)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ btrfs_warn(fs_info,
+++++++++++++ ++ ++++++++"block group %llu read-only transition waiting for stripe runs that do not drain",
+++++++++++++ ++ ++++++++ bg->start);
+++++++++++++ ++ ++++++++ btrfs_dump_open_stripe_runs(bg);
+++++++++++++ ++ ++++++++ warned = true;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (BTRFS_FS_ERROR(fs_info) && !bg_open_stripes_settled(bg, U64_MAX))
+++++++++++++ ++ ++++++++ btrfs_warn(fs_info,
+++++++++++++ ++ ++++++++"block group %llu read-only with stripe runs still in flight after a filesystem error; not waiting for them",
+++++++++++++ ++ ++++++++ bg->start);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * The filesystem has gone into error: wake every task waiting for a block
+++++++++++++ ++ ++++++++ * group's stripe runs to drain, so the waits that give up on an error
+++++++++++++ ++ ++++++++ * (btrfs_retire_block_group_stripes()) notice it. A run's own completion
+++++++++++++ ++ ++++++++ * reports may never arrive now.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_wake_open_stripe_waiters(struct btrfs_fs_info *fs_info)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * The caller set fs_error without a lock, and wake_up_var() checks
+++++++++++++ ++ ++++++++ * for waiters without one: order the store before that check, or a
+++++++++++++ ++ ++++++++ * waiter that has just tested BTRFS_FS_ERROR is missed. Pairs with
+++++++++++++ ++ ++++++++ * the barrier in prepare_to_wait_event().
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ smp_mb();
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
+++++++++++++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Teardown: free every remaining stripe run and drop each group's
+++++++++++++ ++ ++++++++ * membership reference. Needed when no final commit retired them: a mount
+++++++++++++ ++ ++++++++ * that fails after log replay allocated tree blocks, or an unmount after a
+++++++++++++ ++ ++++++++ * transaction abort. IO is quiesced by now, so a run still counting
+++++++++++++ ++ ++++++++ * inflight bytes is reported and freed anyway.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Ordered extents point at their runs (ordered->stripe_run), so this relies
+++++++++++++ ++ ++++++++ * on none being left: at unmount every inode, and with it every ordered
+++++++++++++ ++ ++++++++ * extent, is gone before close_ctree() runs, delayed iputs are drained, and
+++++++++++++ ++ ++++++++ * on both paths the workers that complete ordered extents are stopped first.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_free_open_stripe_runs(struct btrfs_fs_info *fs_info)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ LIST_HEAD(bgs);
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg, *tmp_bg;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ list_splice_init(&fs_info->open_stripe_bgs, &bgs);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ list_for_each_entry_safe(bg, tmp_bg, &bgs, open_stripe_bg_list) {
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run, *tmp;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ list_del_init(&bg->open_stripe_bg_list);
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry_safe(run, tmp, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (run->inflight_bytes)
+++++++++++++ ++ ++++++++ btrfs_warn(fs_info,
+++++++++++++ ++ ++++++++"stripe run %llu-%llu in block group %llu released at teardown with %llu bytes still in flight",
+++++++++++++ ++ ++++++++ run->start, run->end, bg->start,
+++++++++++++ ++ ++++++++ run->inflight_bytes);
+++++++++++++ ++ ++++++++ list_del(&run->list);
+++++++++++++ ++ ++++++++ bitmap_free(run->live);
+++++++++++++ ++ ++++++++ bitmap_free(run->written);
+++++++++++++ ++ ++++++++ kfree(run);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ memset(bg->open_stripe, 0, sizeof(bg->open_stripe));
+++++++++++++ ++ ++++++++ bg->stripe_open_remainder = 0;
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Close and drain the metadata stripe runs once every tree block this
+++++++++++++ ++ ++++++++ * transaction allocated has reached the disk.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * btrfs_retire_open_stripes() runs earlier in the commit and skips these
+++++++++++++ ++ ++++++++ * runs, so they stay open for the commit's own tree block allocations.
+++++++++++++ ++ ++++++++ * Closing earlier would strand blocks the commit has yet to allocate;
+++++++++++++ ++ ++++++++ * draining earlier would wait for writes the commit has yet to issue.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_retire_meta_stripes(struct btrfs_fs_info *fs_info, u64 transid)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ LIST_HEAD(retire_list);
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * No option test: a runtime disable leaves the running transaction's
+++++++++++++ ++ ++++++++ * metadata runs open, and the commit that follows still has to close
+++++++++++++ ++ ++++++++ * and drain them or their block groups never drop their references.
+++++++++++++ ++ ++++++++ * The list is empty when nothing is open, so this costs nothing on a
+++++++++++++ ++ ++++++++ * filesystem that never had the policy.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
+++++++++++++ ++ ++++++++ list_add_tail(&bg->open_stripe_retire_list, &retire_list);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ while (!list_empty(&retire_list)) {
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ bool again = true;
+++++++++++++ ++ ++++++++ bool warned = false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ bg = list_first_entry(&retire_list, struct btrfs_block_group,
+++++++++++++ ++ ++++++++ open_stripe_retire_list);
+++++++++++++ ++ ++++++++ list_del_init(&bg->open_stripe_retire_list);
+++++++++++++ ++ ++++++++ if (!(bg->flags & (BTRFS_BLOCK_GROUP_METADATA |
+++++++++++++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ while (again) {
+++++++++++++ ++ ++++++++ u64 tail_start;
+++++++++++++ ++ ++++++++ u64 tail_len = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ again = false;
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (!btrfs_stripe_run_class_is_meta(run->class) ||
+++++++++++++ ++ ++++++++ !run->open)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * The running transaction's runs are not ours
+++++++++++++ ++ ++++++++ * to close: their tree blocks are written, and
+++++++++++++ ++ ++++++++ * their stripes settled, by that transaction's
+++++++++++++ ++ ++++++++ * own commit.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (run->transid > transid)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ for (int band = 0;
+++++++++++++ ++ ++++++++ band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+++++++++++++ ++ ++++++++ if (bg->open_stripe[BTRFS_STRIPE_RUN_META][band] == run)
+++++++++++++ ++ ++++++++ bg->open_stripe[BTRFS_STRIPE_RUN_META][band] = NULL;
+++++++++++++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++++ ++ ++++++++ again = true;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (tail_len)
+++++++++++++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * The closes above shrank the group's open-run remainder;
+++++++++++++ ++ ++++++++ * fold the delta into the space_info now. While the policy
+++++++++++++ ++ ++++++++ * is on the next allocation from this group would carry it,
+++++++++++++ ++ ++++++++ * but after a runtime disable nothing allocates here again
+++++++++++++ ++ ++++++++ * and the stale remainder holds metadata back for good.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Every tree block is on disk, so end_bbio_meta_write() has
+++++++++++++ ++ ++++++++ * reported all of them and this normally does not wait. A run
+++++++++++++ ++ ++++++++ * that does not drain has a tree block that was freed without
+++++++++++++ ++ ++++++++ * reporting to its run, an accounting bug: say which runs
+++++++++++++ ++ ++++++++ * once, and keep waiting. Only a filesystem error ends the
+++++++++++++ ++ ++++++++ * wait early.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ while (!wait_var_event_timeout(&bg->open_stripe_runs,
+++++++++++++ ++ ++++++++ !bg_has_undrained_meta_runs(bg, transid) ||
+++++++++++++ ++ ++++++++ BTRFS_FS_ERROR(fs_info), 60 * HZ)) {
+++++++++++++ ++ ++++++++ if (warned)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ btrfs_warn(fs_info,
+++++++++++++ ++ ++++++++"metadata stripe runs in block group %llu do not drain; a tree block was freed without reporting to its run",
+++++++++++++ ++ ++++++++ bg->start);
+++++++++++++ ++ ++++++++ btrfs_dump_open_stripe_runs(bg);
+++++++++++++ ++ ++++++++ warned = true;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Drained and no longer needed for padding or merging. */
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ while (true) {
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *next = NULL;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class) &&
+++++++++++++ ++ ++++++++ open_stripe_run_drained(run)) {
+++++++++++++ ++ ++++++++ next = run;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (!next)
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ free_open_stripe_run(bg, next);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Should this tree block be left dirty for the transaction commit to write?
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * A stripe of tree blocks goes out as one full-stripe write only if all of
+++++++++++++ ++ ++++++++ * its blocks are written in the same pass. Opportunistic writeback (memory
+++++++++++++ ++ ++++++++ * reclaim, the periodic timer) does not know about stripes and would split
+++++++++++++ ++ ++++++++ * it into read-modify-writes, so it is declined here, as zoned mode does in
+++++++++++++ ++ ++++++++ * btrfs_check_meta_write_pointer(), and a commit is requested instead.
+++++++++++++ ++ ++++++++ * WB_SYNC_ALL is never deferred, so a caller that needs the pages written
+++++++++++++ ++ ++++++++ * can always get them written.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++bool btrfs_defer_stripe_meta_write(struct btrfs_fs_info *fs_info,
+++++++++++++ ++ ++++++++ struct extent_buffer *eb,
+++++++++++++ ++ ++++++++ const struct writeback_control *wbc)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ bool defer;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ if (wbc->sync_mode == WB_SYNC_ALL)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, eb->start);
+++++++++++++ ++ ++++++++ if (!bg)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ defer = btrfs_is_stripe_meta_bg(bg);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (defer) {
+++++++++++++ ++ ++++++++ set_bit(BTRFS_FS_NEED_TRANS_COMMIT, &fs_info->flags);
+++++++++++++ ++ ++++++++ wake_up_process(fs_info->transaction_kthread);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ return defer;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Commit-time retirement: close every data stripe run opened before this
+++++++++++++ ++ ++++++++ * call and wait for its IO to complete. Metadata runs are retired later by
+++++++++++++ ++ ++++++++ * btrfs_retire_meta_stripes(), and persistent NOCOW runs survive. Must run
+++++++++++++ ++ ++++++++ * while the committing transaction still accepts joins (before
+++++++++++++ ++ ++++++++ * TRANS_STATE_COMMIT_DOING): tasks holding unreported run bytes can block on
+++++++++++++ ++ ++++++++ * a join, and this drain waits for their IO. Records the bumped sequence in
+++++++++++++ ++ ++++++++ * @trans; ordered extents from runs with open_seq >= it defer their file
+++++++++++++ ++ ++++++++ * extents to the next transaction. Calls are serialized by the transaction
+++++++++++++ ++ ++++++++ * commit; @trans is NULL for the final cleanup call.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info,
+++++++++++++ ++ ++++++++ struct btrfs_transaction *trans)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ LIST_HEAD(retire_list);
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ u64 seq;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ seq = ++fs_info->stripe_retire_seq;
+++++++++++++ ++ ++++++++ if (trans)
+++++++++++++ ++ ++++++++ trans->stripe_retire_seq = seq;
+++++++++++++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
+++++++++++++ ++ ++++++++ list_add_tail(&bg->open_stripe_retire_list, &retire_list);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ list_for_each_entry(bg, &retire_list, open_stripe_retire_list)
+++++++++++++ ++ ++++++++ close_block_group_stripe_runs(bg, seq);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Parked partial-stripe write rbios (see raid56.c) hold bios whose
+++++++++++++ ++ ++++++++ * write_done the drain below waits for; kick them all down first.
+++++++++++++ ++ ++++++++ * A racing park that saw its run still open lands after this flush
+++++++++++++ ++ ++++++++ * and is bounded by the park timer, not by us.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, 0, U64_MAX, false);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ while (!list_empty(&retire_list)) {
+++++++++++++ ++ ++++++++ bg = list_first_entry(&retire_list, struct btrfs_block_group,
+++++++++++++ ++ ++++++++ open_stripe_retire_list);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Same escape as btrfs_retire_block_group_stripes(): on an
+++++++++++++ ++ ++++++++ * errored filesystem the bytes in flight may never be
+++++++++++++ ++ ++++++++ * reported, and this wait sits in the transaction commit and
+++++++++++++ ++ ++++++++ * in the teardown call from close_ctree().
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ wait_var_event(&bg->open_stripe_runs,
+++++++++++++ ++ ++++++++ bg_open_stripes_settled(bg, seq) ||
+++++++++++++ ++ ++++++++ BTRFS_FS_ERROR(fs_info));
+++++++++++++ ++ ++++++++ if (BTRFS_FS_ERROR(fs_info) && !bg_open_stripes_settled(bg, seq))
+++++++++++++ ++ ++++++++ btrfs_warn(fs_info,
+++++++++++++ ++ ++++++++"block group %llu has stripe runs still in flight at retirement after a filesystem error; not waiting for them",
+++++++++++++ ++ ++++++++ bg->start);
+++++++++++++ ++ ++++++++ list_del_init(&bg->open_stripe_retire_list);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Drop the membership if the group has no runs left. */
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ if (list_empty(&bg->open_stripe_runs) &&
+++++++++++++ ++ ++++++++ !list_empty(&bg->open_stripe_bg_list)) {
+++++++++++++ ++ ++++++++ list_del_init(&bg->open_stripe_bg_list);
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ } else {
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Commit-time recompute of stripe_unusable for every stripe_alloc block group
+++++++++++++ ++ ++++++++ * whose free space changed this transaction. Trapped free space only settles
+++++++++++++ ++ ++++++++ * at commit -- partially filled stripes are returned by the retire above and
+++++++++++++ ++ ++++++++ * deleted extents are unpinned into the free space cache -- so this is where
+++++++++++++ ++ ++++++++ * the counter can be made authoritative. Called after btrfs_finish_extent_commit
+++++++++++++ ++ ++++++++ * so the free space cache reflects this transaction's frees.
+++++++++++++ ++ ++++++++ */
++++++++++++++ +++++++++++static void reloc_reserve_resync(struct btrfs_block_group *bg);
++++++++++++++ +++++++++++
+++++++++++++ ++ ++++++++void btrfs_scan_stripe_unusable(struct btrfs_fs_info *fs_info, bool force)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_space_info *sinfo;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ list_for_each_entry(sinfo, &fs_info->space_info, list) {
+++++++++++++ ++ ++++++++ u64 total = 0;
+++++++++++++ ++ ++++++++ u64 total_claimable = 0;
+++++++++++++ ++ ++++++++ u64 total_claimable_reloc = 0;
+++++++++++++ ++ ++++++++ u64 total_stranded = 0;
+++++++++++++ ++ ++++++++ u64 largest = 0;
+++++++++++++ ++ ++++++++ u64 unit = 0;
+++++++++++++ ++ ++++++++ u32 nr_rw = 0;
+++++++++++++ ++ ++++++++ int raid;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Data groups under stripe_alloc and metadata groups under
+++++++++++++ ++ ++++++++ * stripe_meta both place extents in whole stripes only, so
+++++++++++++ ++ ++++++++ * both carry trapped free space. Mixed groups are refused.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (!(sinfo->flags & (BTRFS_BLOCK_GROUP_DATA |
+++++++++++++ ++ ++++++++ BTRFS_BLOCK_GROUP_METADATA |
+++++++++++++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ down_read(&sinfo->groups_sem);
+++++++++++++ ++ ++++++++ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!(btrfs_raid_array[raid].bg_flag &
+++++++++++++ ++ ++++++++ BTRFS_BLOCK_GROUP_RAID56_MASK))
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ list_for_each_entry(bg, &sinfo->block_groups[raid],
+++++++++++++ ++ ++++++++ list) {
+++++++++++++ ++ ++++++++ bool dirty = test_and_clear_bit(
+++++++++++++ ++ ++++++++ BLOCK_GROUP_FLAG_STRIPE_UNUSABLE_DIRTY,
+++++++++++++ ++ ++++++++ &bg->runtime_flags);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * @force rescans every armed group regardless of
+++++++++++++ ++ ++++++++ * the dirty flag (debug: isolates a missed
+++++++++++++ ++ ++++++++ * dirty-marking from a genuine cache difference).
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (force || dirty)
+++++++++++++ ++ ++++++++ btrfs_block_group_rescan_stripe_unusable(bg);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Sum the authoritative per-group values into the space_info
+++++++++++++ ++ ++++++++ * total so it cannot drift from repeated arm/disarm/rescan
+++++++++++++ ++ ++++++++ * deltas. Disarmed (read-only) groups are excluded; their
+++++++++++++ ++ ++++++++ * free space is already accounted as read-only. Every update
+++++++++++++ ++ ++++++++ * of a group's values also updates the total under
+++++++++++++ ++ ++++++++ * sinfo->lock, so summing under it gives totals that match
+++++++++++++ ++ ++++++++ * the groups: claims, arms and disarms that race with the
+++++++++++++ ++ ++++++++ * scan are neither lost nor counted twice.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ spin_lock(&sinfo->lock);
+++++++++++++ ++ ++++++++ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!(btrfs_raid_array[raid].bg_flag &
+++++++++++++ ++ ++++++++ BTRFS_BLOCK_GROUP_RAID56_MASK))
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ list_for_each_entry(bg, &sinfo->block_groups[raid],
+++++++++++++ ++ ++++++++ list) {
+++++++++++++ ++ ++++++++ if (bg->stripe_unusable_ready) {
+++++++++++++ ++ ++++++++ total += bg->stripe_unusable;
+++++++++++++ ++ ++++++++ total_stranded += READ_ONCE(bg->stripe_stranded);
+++++++++++++ ++ ++++++++ total_claimable += bg->stripe_claimable;
++++++++++++++ +++++++++++ /* a held group's are in bytes_reloc_reserve */
++++++++++++++ +++++++++++ if (bg->start == READ_ONCE(fs_info->data_reloc_bg) &&
++++++++++++++ +++++++++++ !test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
++++++++++++++ +++++++++++ &bg->runtime_flags))
+++++++++++++ ++ ++++++++ total_claimable_reloc +=
+++++++++++++ ++ ++++++++ bg->stripe_claimable;
+++++++++++++ ++ ++++++++ nr_rw++;
+++++++++++++ ++ ++++++++ largest = max(largest, bg->length);
++++++++++++++ +++++++++++ if (test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
++++++++++++++ +++++++++++ &bg->runtime_flags)) {
++++++++++++++ +++++++++++ spin_lock(&bg->lock);
++++++++++++++ +++++++++++ reloc_reserve_resync(bg);
++++++++++++++ +++++++++++ spin_unlock(&bg->lock);
++++++++++++++ +++++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ unit = max(unit, bg->full_stripe_len);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ up_read(&sinfo->groups_sem);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ sinfo->bytes_stripe_unusable = total;
+++++++++++++ ++ ++++++++ sinfo->bytes_stripe_claimable = total_claimable;
+++++++++++++ ++ ++++++++ sinfo->bytes_stripe_claimable_reloc = total_claimable_reloc;
+++++++++++++ ++ ++++++++ sinfo->bytes_stripe_stranded = total_stranded;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Not if the option went off since the check above: the
+++++++++++++ ++ ++++++++ * disable path has cleared the margin and reserve, and no
+++++++++++++ ++ ++++++++ * later scan would clear them again.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (!(sinfo->flags & BTRFS_BLOCK_GROUP_DATA) &&
+++++++++++++ ++ ++++++++ btrfs_test_opt(fs_info, STRIPE_ALLOC)) {
+++++++++++++ ++ ++++++++ u64 reserve = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * stripe_meta. A tree block is reserved in bytes but
+++++++++++++ ++ ++++++++ * placed in whole stripes: the run it would join is
+++++++++++++ ++ ++++++++ * closed at the transaction boundary, so each group
+++++++++++++ ++ ++++++++ * claims at least one fresh stripe per transaction.
+++++++++++++ ++ ++++++++ * Pre-pay two transactions' worth of such claims, or
+++++++++++++ ++ ++++++++ * the last claimable stripes go to claims made for
+++++++++++++ ++ ++++++++ * reservations admitted earlier and an admitted tree
+++++++++++++ ++ ++++++++ * block finds none, which aborts the transaction.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ sinfo->bytes_stripe_margin = 2ULL * nr_rw * unit;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * COW frees trap stripes (a stripe is claimable only
+++++++++++++ ++ ++++++++ * once every block in it is dead), and only relocation
+++++++++++++ ++ ++++++++ * packs live blocks back into whole stripes, which
+++++++++++++ ++ ++++++++ * needs room elsewhere for the moved group. With more
+++++++++++++ ++ ++++++++ * than one group, hold back one group's worth, capped
+++++++++++++ ++ ++++++++ * at a quarter of the space_info, from all but
+++++++++++++ ++ ++++++++ * relocation. Accounting only: a committing
+++++++++++++ ++ ++++++++ * transaction may still allocate here.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (nr_rw > 1 && unit)
+++++++++++++ ++ ++++++++ reserve = stripe_round_up(min(largest,
+++++++++++++ ++ ++++++++ sinfo->total_bytes / 4),
+++++++++++++ ++ ++++++++ unit);
+++++++++++++ ++ ++++++++ sinfo->bytes_stripe_reserve = reserve;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * The rescan usually lowers the counter (mid-transaction
+++++++++++++ ++ ++++++++ * incremental adds overcount conservatively); admission
+++++++++++++ ++ ++++++++ * tickets may be waiting on exactly that headroom.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ btrfs_try_granting_tickets(sinfo);
+++++++++++++ ++ ++++++++ spin_unlock(&sinfo->lock);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Does stripe-exclusive allocation force COW for an in-place (nocow or
+++++++++++++ ++ ++++++++ * prealloc) write at @bytenr? True for any raid56 data group under
+++++++++++++ ++ ++++++++ * stripe_alloc, since an in-place write could tear a stripe holding other
+++++++++++++ ++ ++++++++ * files' committed extents. The caller may still allow the write when
+++++++++++++ ++ ++++++++ * btrfs_stripe_nocow_writable() proves the stripes belong to the inode.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++bool btrfs_stripe_alloc_forces_cow(struct btrfs_fs_info *fs_info, u64 bytenr)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ bool ret = false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
+++++++++++++ ++ ++++++++ if (bg) {
+++++++++++++ ++ ++++++++ ret = btrfs_is_stripe_alloc_bg(bg);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ return ret;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Write-hole invariant checker, called for every raid56 write (full-stripe
+++++++++++++ ++ ++++++++ * and sub-stripe alike). Under stripe_alloc and stripe_meta a stripe may
+++++++++++++ ++ ++++++++ * only be written while an open or draining run covers it; a write outside
+++++++++++++ ++ ++++++++ * every run means a retired stripe is being rewritten or an allocation
+++++++++++++ ++ ++++++++ * bypassed the policy.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Groups that hosted RELOC or NOCOW runs are skipped: those extents are
+++++++++++++ ++ ++++++++ * legitimately overwritten in place after their runs drain.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_stripe_check_write(struct btrfs_fs_info *fs_info,
+++++++++++++ ++ ++++++++ u64 full_stripe_start, bool sub_stripe)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ unsigned long flags;
+++++++++++++ ++ ++++++++ bool live = false;
+++++++++++++ ++ ++++++++ u64 fsl;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ /* the legacy allocator's writes are still landing after a runtime enable */
+++++++++++++ ++ ++++++++ if (test_bit(BTRFS_FS_STRIPE_ALLOC_ENABLING, &fs_info->flags))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, full_stripe_start);
+++++++++++++ ++ ++++++++ if (!bg)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Metadata is checked on the same terms once stripe_meta places it:
+++++++++++++ ++ ++++++++ * its runs stay open across the commit's tree building but are closed
+++++++++++++ ++ ++++++++ * and drained before the commit ends, so a tree block write outside
+++++++++++++ ++ ++++++++ * every live run means a stripe was revisited after the transaction
+++++++++++++ ++ ++++++++ * that filled it finished -- the same violation as for data.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if ((!btrfs_is_stripe_alloc_bg(bg) && !btrfs_is_stripe_meta_bg(bg)) ||
+++++++++++++ ++ ++++++++ test_bit(BLOCK_GROUP_FLAG_STRIPE_RELOC_USED, &bg->runtime_flags) ||
+++++++++++++ ++ ++++++++ test_bit(BLOCK_GROUP_FLAG_STRIPE_NOCOW_USED, &bg->runtime_flags))
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ fsl = bg->full_stripe_len;
+++++++++++++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++++ ++ ++++++++ if (full_stripe_start < run->end &&
+++++++++++++ ++ ++++++++ full_stripe_start + fsl > run->start) {
+++++++++++++ ++ ++++++++ live = true;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++++ ++ ++++++++ WARN_ONCE(!live,
+++++++++++++ ++ ++++++++"btrfs: %s write to stripe %llu (block group %llu) outside any live stripe run: write hole window violated",
+++++++++++++ ++ ++++++++ sub_stripe ? "sub-stripe" : "full-stripe",
+++++++++++++ ++ ++++++++ full_stripe_start, bg->start);
+++++++++++++ ++ ++++++++out:
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++#endif
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Drop the dedication of a block group to data relocation. Shared by the
+++++++++++++ ++ ++++++++ * zoned allocator and the stripe allocation policy; both dedicate one
+++++++++++++ ++ ++++++++ * block group at a time via fs_info->data_reloc_bg.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ bool cleared = false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->relocation_bg_lock);
+++++++++++++ ++ ++++++++ if (fs_info->data_reloc_bg == bg->start) {
+++++++++++++ ++ ++++++++ fs_info->data_reloc_bg = 0;
+++++++++++++ ++ ++++++++ cleared = true;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->relocation_bg_lock);
+++++++++++++ ++ ++++++++ if (cleared && bg->space_info) {
+++++++++++++ ++ ++++++++ spin_lock(&bg->space_info->lock);
+++++++++++++ ++ ++++++++ bg->space_info->bytes_stripe_claimable_reloc = 0;
+++++++++++++ ++ ++++++++ spin_unlock(&bg->space_info->lock);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Release the block group currently dedicated to data relocation, closing
+++++++++++++ ++ ++++++++ * its open stripe runs so ordinary allocations can use the remaining fully
+++++++++++++ ++ ++++++++ * free stripes. Used by the stripe allocation policy when relocation of a
+++++++++++++ ++ ++++++++ * block group finishes.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_release_data_reloc_bg(struct btrfs_fs_info *fs_info)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ u64 bytenr;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock(&fs_info->relocation_bg_lock);
+++++++++++++ ++ ++++++++ bytenr = fs_info->data_reloc_bg;
+++++++++++++ ++ ++++++++ fs_info->data_reloc_bg = 0;
+++++++++++++ ++ ++++++++ spin_unlock(&fs_info->relocation_bg_lock);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!bytenr)
+++++++++++++ ++ ++++++++ goto release;
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
+++++++++++++ ++ ++++++++ if (!bg)
+++++++++++++ ++ ++++++++ goto release;
+++++++++++++ ++ ++++++++ if (bg->space_info) {
+++++++++++++ ++ ++++++++ spin_lock(&bg->space_info->lock);
+++++++++++++ ++ ++++++++ bg->space_info->bytes_stripe_claimable_reloc = 0;
+++++++++++++ ++ ++++++++ spin_unlock(&bg->space_info->lock);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ btrfs_close_bg_open_stripes(bg);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++release:
+++++++++++++ ++ ++++++++ btrfs_release_stripe_reloc_runs(fs_info);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
struct btrfs_caching_control *btrfs_get_caching_control(
struct btrfs_block_group *cache)
{
* btrfs_can_overcommit check here, and we need to pass in
* BTRFS_RESERVE_NO_FLUSH to give ourselves the most amount of
* leeway to allow us to mark this block group as read only.
- if (btrfs_can_overcommit(sinfo, num_bytes, BTRFS_RESERVE_NO_FLUSH))
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * As for data above: a stripe_meta group's trapped bytes are
+++++++++++++ ++ ++++++++ * already in the space_info total, do not re-provide them.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ u64 trapped = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (READ_ONCE(cache->stripe_unusable_ready))
+++++++++++++ ++ ++++++++ trapped = min(READ_ONCE(cache->stripe_unusable), num_bytes);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * The stripe_meta reserve is the room a group's live blocks
+++++++++++++ ++ ++++++++ * move into when it is relocated, and going read-only is the
+++++++++++++ ++ ++++++++ * first step of exactly that: count the reserve as available
+++++++++++++ ++ ++++++++ * here, or the guard refuses every relocation at the edge by
+++++++++++++ ++ ++++++++ * the width of the reserve.
++++++++++++ ++ ++++++++ */
------------ -- -------- if (btrfs_can_overcommit(sinfo, num_bytes, BTRFS_RESERVE_NO_FLUSH))
+++++++++++++ ++ ++++++++ if (btrfs_can_overcommit(sinfo, num_bytes - trapped,
+++++++++++++ ++ ++++++++ BTRFS_RESERVE_NO_FLUSH, true))
+++++++++++++ ++ ++++++++ ret = 0;
++++++++++++++ +++++++++++ /*
++++++++++++++ +++++++++++ * Or the relocation reserve can take this group's live tree
++++++++++++++ +++++++++++ * blocks: a failed tree block allocation outside the reserve
++++++++++++++ +++++++++++ * falls back into it. Only with a reserve and something to
++++++++++++++ +++++++++++ * move: an empty group would pass "0 >= 0" with the reserve
++++++++++++++ +++++++++++ * off. Data gets no such exemption: its test above fails only
++++++++++++++ +++++++++++ * while bytes_may_use depends on this group's free space, and
++++++++++++++ +++++++++++ * data writeback has no reserve to fall back on.
++ + */
++++++++++++++ +++++++++++ else if (sinfo->bytes_reloc_reserve &&
++++++++++++++ +++++++++++ cache->used + cache->reserved &&
++++++++++++++ +++++++++++ sinfo->bytes_reloc_reserve >=
++++++++++++++ +++++++++++ cache->used + cache->reserved)
+ + ret = 0;
}
if (!ret) {
list_add_tail(&cache->ro_list, &sinfo->ro_bgs);
}
out:
++++++++++++++ +++++++++++ if (ret)
++++++++++++++ +++++++++++ btrfs_reloc_reserve_rejoin(cache);
spin_unlock(&cache->lock);
spin_unlock(&sinfo->lock);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * A read-only group's free space (trapped included) is now accounted
+++++++++++++ ++ ++++++++ * as read-only and excluded from statfs; drop its stripe_unusable
+++++++++++++ ++ ++++++++ * contribution so it is not subtracted twice. Idempotent, so repeated
+++++++++++++ ++ ++++++++ * ro increments are harmless.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (!ret) {
+++++++++++++ ++ ++++++++ btrfs_block_group_disarm_stripe_unusable(cache);
+++++++++++++ ++ ++++++++ btrfs_stripe_unmanaged_sync(cache);
+++++++++++++ ++ ++++++++ }
if (ret == -ENOSPC && btrfs_test_opt(cache->fs_info, ENOSPC_DEBUG)) {
btrfs_info(cache->fs_info,
"unable to make block group %llu ro", cache->start);
spin_unlock(&bg->lock);
trace_btrfs_reclaim_block_group(bg);
------------ -- -------- ret = btrfs_relocate_chunk(fs_info, bg->start, false);
++++++++++++ ++ ++++++++ btrfs_debug(fs_info,
- "reclaim: relocating block group %llu flags 0x%llx length %llu used %llu reserved %llu",
- bg->start, bg->flags,
- bg->length, used, reserved);
- ret = btrfs_relocate_chunk(fs_info, bg->start, false);
+++++++++++++ ++ ++++++++"reclaim: %s block group %llu flags 0x%llx length %llu used %llu reserved %llu trapped %llu",
+++++++++++++ ++ ++++++++ compact ? "compacting" : "relocating", bg->start, bg->flags,
+++++++++++++ ++ ++++++++ bg->length, used, reserved, trapped);
+++++++++++++ ++ ++++++++ if (compact) {
+++++++++++++ ++ ++++++++ ret = btrfs_compact_chunk(fs_info, bg->start, &compacted);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Compacted, the group stays: drop our read-only hold. It
+++++++++++++ ++ ++++++++ * gave back trapped space, not a chunk; what moved is counted
+++++++++++++ ++ ++++++++ * in stripe_run_stats.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (!ret && compacted) {
+++++++++++++ ++ ++++++++ btrfs_dec_block_group_ro(bg);
+++++++++++++ ++ ++++++++ used = 0;
+++++++++++++ ++ ++++++++ reserved = 0;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ } else {
+++++++++++++ ++ ++++++++ ret = btrfs_relocate_chunk(fs_info, bg->start, false);
+++++++++++++ ++ ++++++++ }
if (btrfs_is_zoned(fs_info) && ret == -EAGAIN) {
btrfs_dec_block_group_ro(bg);
btrfs_debug(fs_info, "deferring reclaim of chunk %llu", bg->start);
if (space_info->total_bytes < old_total)
btrfs_set_periodic_reclaim_ready(space_info, true);
spin_unlock(&space_info->lock);
------------ -- -------- if (!ret)
++++++++++++ ++ ++++++++ if (!ret) {
(*reclaimed)++;
- btrfs_debug(fs_info, "reclaim: relocated block group %llu",
- bg->start);
+++++++++++++ ++ ++++++++ btrfs_debug(fs_info, "reclaim: %s block group %llu",
+++++++++++++ ++ ++++++++ compacted ? "compacted" : "relocated", bg->start);
++++++++++++ ++ ++++++++ }
+
+ return ret;
+ }
+
++++++++++++ ++ ++++++++/*
++++++++++++ ++ ++++++++ * Reclaim gives up on a space_info for a while after this many reclaims in a
- * row failed with -ENOSPC. Relocating a group needs room for
+++++++++++++ ++ ++++++++ * row failed with -ENOSPC. Relocating or compacting a group needs room for
++++++++++++ ++ ++++++++ * its live data (and metadata reservations for moving it); at the fill edge
++++++++++++ ++ ++++++++ * there is none, every attempt fails after doing its work, and retrying each
++++++++++++ ++ ++++++++ * failed group on every worker run held the balance exclusive operation for
++++++++++++ ++ ++++++++ * over an hour while writers starved (octopus P1, bhive 2026-10-20).
++++++++++++ ++ ++++++++ */
++++++++++++ ++ ++++++++#define BTRFS_RECLAIM_ENOSPC_STREAK 3
++++++++++++ ++ ++++++++#define BTRFS_RECLAIM_BACKOFF (10 * 60 * HZ)
++++++++++++ ++ ++++++++
++++++++++++ ++ ++++++++/* Free space that a retried reclaim could use: free in groups, unallocated. */
++++++++++++ ++ ++++++++static u64 reclaim_free_bytes(const struct btrfs_space_info *space_info)
++++++++++++ ++ ++++++++{
++++++++++++ ++ ++++++++ return space_info->total_bytes - min(space_info->total_bytes,
++++++++++++ ++ ++++++++ space_info->bytes_used) +
++++++++++++ ++ ++++++++ atomic64_read(&space_info->fs_info->free_chunk_space);
++++++++++++ ++ ++++++++}
++++++++++++ ++ ++++++++
++++++++++++ ++ ++++++++/*
++++++++++++ ++ ++++++++ * Is reclaim backing off for this space_info? The backoff ends when its
++++++++++++ ++ ++++++++ * time is up, when a chunk was added or removed, or when free space grew by
++++++++++++ ++ ++++++++ * what the last failed group needed.
++++++++++++ ++ ++++++++ */
++++++++++++ ++ ++++++++bool btrfs_reclaim_backoff_active(struct btrfs_space_info *space_info)
++++++++++++ ++ ++++++++{
++++++++++++ ++ ++++++++ bool active = false;
++++++++++++ ++ ++++++++
++++++++++++ ++ ++++++++ spin_lock(&space_info->lock);
++++++++++++ ++ ++++++++ if (space_info->reclaim_backoff_until) {
++++++++++++ ++ ++++++++ active = time_before(jiffies, space_info->reclaim_backoff_until) &&
++++++++++++ ++ ++++++++ space_info->total_bytes == space_info->reclaim_backoff_total &&
++++++++++++ ++ ++++++++ reclaim_free_bytes(space_info) < space_info->reclaim_backoff_free;
++++++++++++ ++ ++++++++ if (!active) {
++++++++++++ ++ ++++++++ space_info->reclaim_backoff_until = 0;
++++++++++++ ++ ++++++++ space_info->reclaim_enospc_streak = 0;
++++++++++++ ++ ++++++++ }
++++++++++++ ++ ++++++++ }
++++++++++++ ++ ++++++++ spin_unlock(&space_info->lock);
++++++++++++ ++ ++++++++ return active;
++++++++++++ ++ ++++++++}
++++++++++++ ++ ++++++++
++++++++++++ ++ ++++++++/*
++++++++++++ ++ ++++++++ * Count a reclaim of @bg that failed with -ENOSPC (@enospc) or succeeded,
++++++++++++ ++ ++++++++ * and start the backoff at the end of a streak of failures. @need is what
++++++++++++ ++ ++++++++ * the group held: free space growing by that much ends the backoff early.
++++++++++++ ++ ++++++++ */
++++++++++++ ++ ++++++++static void reclaim_note_result(struct btrfs_block_group *bg, bool enospc, u64 need)
++++++++++++ ++ ++++++++{
++++++++++++ ++ ++++++++ struct btrfs_space_info *space_info = bg->space_info;
++++++++++++ ++ ++++++++ u32 streak;
++++++++++++ ++ ++++++++
++++++++++++ ++ ++++++++ spin_lock(&space_info->lock);
++++++++++++ ++ ++++++++ if (!enospc) {
++++++++++++ ++ ++++++++ space_info->reclaim_enospc_streak = 0;
++++++++++++ ++ ++++++++ spin_unlock(&space_info->lock);
++++++++++++ ++ ++++++++ return;
++++++++++++ ++ ++++++++ }
++++++++++++ ++ ++++++++ streak = ++space_info->reclaim_enospc_streak;
++++++++++++ ++ ++++++++ if (streak >= BTRFS_RECLAIM_ENOSPC_STREAK) {
++++++++++++ ++ ++++++++ space_info->reclaim_backoff_until = (jiffies + BTRFS_RECLAIM_BACKOFF) ?: 1;
++++++++++++ ++ ++++++++ space_info->reclaim_backoff_total = space_info->total_bytes;
++++++++++++ ++ ++++++++ space_info->reclaim_backoff_free = reclaim_free_bytes(space_info) +
++++++++++++ ++ ++++++++ max_t(u64, need, SZ_16M);
++++++++++++ ++ ++++++++ space_info->reclaim_backoffs++;
++++++++++++ ++ ++++++++ }
++++++++++++ ++ ++++++++ spin_unlock(&space_info->lock);
++++++++++++ ++ ++++++++ if (streak >= BTRFS_RECLAIM_ENOSPC_STREAK)
++++++++++++ ++ ++++++++ btrfs_debug(bg->fs_info,
++++++++++++ ++ ++++++++"reclaim: %u reclaims failed with ENOSPC (last block group %llu), backing off flags 0x%llx",
++++++++++++ ++ ++++++++ streak, bg->start, space_info->flags);
++++++++++++ ++ ++++++++}
++++++ +++++ ++ ++++++++
- return ret;
+++++++++++++++++ ++++++++/* A refused operation's retry window is open: see btrfs_exclop_start(). */
+++++++++++++++++ ++++++++static bool reclaim_yielding(const struct btrfs_fs_info *fs_info)
+++++++++++++++++ ++++++++{
+++++++++++++++++ ++++++++ const unsigned long until = READ_ONCE(fs_info->reclaim_yield_until);
+++++++++++++++++ ++++++++
+++++++++++++++++ ++++++++ return until && time_before(jiffies, until);
+++++++++++++++++ ++++++++}
+++++++++++++++++ ++++++++
+++++++++++++++++ ++++++++/*
+++++++++++++++++ ++++++++ * Unmounting, or read-only after an error: every group taken from here on is
+++++++++++++++++ ++++++++ * dropped (see btrfs_need_cleaner_sleep()), so do not wait out a gap for them.
+++++++++++++++++ ++++++++ */
+++++++++++++++++ ++++++++static bool reclaim_stopping(const struct btrfs_fs_info *fs_info)
+++++++++++++++++ ++++++++{
+++++++++++++++++ ++++++++ return btrfs_need_cleaner_sleep(fs_info) || BTRFS_FS_ERROR(fs_info);
++++++ ++++++++++ ++++++++}
++++++ ++++++++++ ++++++++
void btrfs_reclaim_block_groups(struct btrfs_fs_info *fs_info, unsigned int limit)
{
struct btrfs_block_group *bg;
LIST_HEAD(retry_list);
int reclaimed = 0;
- /* Relocation cannot proceed on an errored filesystem. */
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Relocation cannot proceed, and the stripe drain it starts with
+++++++++++++ ++ ++++++++ * would wait for IO that will not complete.
+++++++++++++ ++ ++++++++ */
++++++++++++ ++ ++++++++ if (BTRFS_FS_ERROR(fs_info))
++++++++++++ ++ ++++++++ return;
++++++++++++ ++ ++++++++
if (!btrfs_should_reclaim(fs_info))
return;
*/
list_sort(NULL, &fs_info->reclaim_bgs, reclaim_bgs_cmp);
while (!list_empty(&fs_info->reclaim_bgs)) {
++++++++++++ ++ ++++++++ u64 need;
+++++++++++++++++ ++++++++ bool handoff = false;
int ret;
bg = list_first_entry(&fs_info->reclaim_bgs,
space_info = bg->space_info;
spin_unlock(&fs_info->unused_bgs_lock);
------------ -- -------- ret = btrfs_reclaim_block_group(bg, &reclaimed);
++++++++++++ ++ ++++++++ need = READ_ONCE(bg->used);
++++++++++++ ++ ++++++++ /*
++++++++++++ ++ ++++++++ * Backing off: drop the group rather than requeue it. The
++++++++++++ ++ ++++++++ * rescans and usage updates queue it again once reclaim may
- * succeed.
+++++++++++++ ++ ++++++++ * succeed. A stripe_reclaim_now request is tried regardless.
++++++++++++ ++ ++++++++ */
- if (btrfs_reclaim_backoff_active(space_info)) {
+++++++++++++ ++ ++++++++ if (!READ_ONCE(bg->stripe_reclaim_forced) &&
+++++++++++++ ++ ++++++++ btrfs_reclaim_backoff_active(space_info)) {
++++++++++++ ++ ++++++++ ret = 0;
++++++++++++ ++ ++++++++ } else {
++++++++++++ ++ ++++++++ int before = reclaimed;
++++++++++++ ++ ++++++++
++++++++++++ ++ ++++++++ ret = btrfs_reclaim_block_group(bg, &reclaimed);
++++++++++++ ++ ++++++++ if (ret == -ENOSPC || reclaimed > before)
++++++++++++ ++ ++++++++ reclaim_note_result(bg, ret == -ENOSPC, need);
+++++++++++++++++ ++++++++ /*
+++++++++++++++++ ++++++++ * It was relocated, or failed to be (an error also
+++++++++++++++++ ++++++++ * comes from setting it read-only, which is quick).
+++++++++++++++++ ++++++++ * Once the limit is reached the loop ends below and
+++++++++++++++++ ++++++++ * releases the operation anyway: no gap.
+++++++++++++++++ ++++++++ */
+++++++++++++++++ ++++++++ handoff = (ret || reclaimed > before) && reclaimed < limit;
++++++++++++ ++ ++++++++ }
++++++++++++ ++ ++++++++ /* An ENOSPC failure is not retried at once; see above. */
if ((btrfs_is_zoned(fs_info) && ret == -EAGAIN) ||
------------ -- -------- (ret && !READ_ONCE(space_info->periodic_reclaim)))
++++++++++++ ++ ++++++++ (ret && ret != -ENOSPC &&
++++++++++++ ++ ++++++++ !READ_ONCE(space_info->periodic_reclaim)))
btrfs_link_bg_list(bg, &retry_list);
btrfs_put_block_group(bg);
mutex_unlock(&fs_info->ro_block_group_mutex);
btrfs_end_transaction(trans);
- if (ret)
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * The group is read-only: close and drain its open stripe runs. An
+++++++++++++ ++ ++++++++ * allocation racing with the read-only transition either installed
+++++++++++++ ++ ++++++++ * its run before this (closed and drained here), or observes ->ro
+++++++++++++ ++ ++++++++ * under the stripe_run_lock and backs out.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (!ret)
+++++++++++++ ++ ++++++++ btrfs_retire_block_group_stripes(cache);
++++++++++++++++++++++++++ else
++++++ +++++++++++++++++++ btrfs_bg_unblock_nocow(cache);
return ret;
}
}
sinfo->bytes_readonly -= btrfs_block_group_available_space(cache);
list_del_init(&cache->ro_list);
+++++++++++++ ++ ++++++++ became_rw = true;
++++++++++++++ +++++++++++ btrfs_reloc_reserve_rejoin(cache);
}
spin_unlock(&cache->lock);
spin_unlock(&sinfo->lock);
spin_unlock(&table->cache_lock);
}
+++++++++++++++++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
+++++++++++++++++ ++++++++/*
+++++++++++++++++ ++++++++ * Test knob (debug/rbio_cache_size): with a small or zero limit, a
+++++++++++++++++ ++++++++ * read-modify-write cannot take its stripe from the cache and must read it,
+++++++++++++++++ ++++++++ * which is what a fault-injecting test needs to see. Lowering it empties the
+++++++++++++++++ ++++++++ * cache, so the new limit holds from the next rbio on.
+++++++++++++++++ ++++++++ */
+++++++++++++++++ ++++++++void btrfs_set_rbio_cache_size(struct btrfs_fs_info *info, unsigned int size)
+++++++++++++++++ ++++++++{
+++++++++++++++++ ++++++++ unsigned int old = READ_ONCE(info->rbio_cache_size);
+++++++++++++++++ ++++++++
+++++++++++++++++ ++++++++ WRITE_ONCE(info->rbio_cache_size, size);
+++++++++++++++++ ++++++++ if (size < old && info->stripe_hash_table)
+++++++++++++++++ ++++++++ btrfs_clear_rbio_cache(info);
+++++++++++++++++ ++++++++}
+++++++++++++++++ ++++++++#endif
+++++++++++++++++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Send every parked rbio down and stop the park timer, before the raid56
+++++++++++++ ++ ++++++++ * workqueue is destroyed: the timer queues its expired rbios there. By
+++++++++++++ ++ ++++++++ * now nothing should be parked; a few passes cover an rbio that parks
+++++++++++++ ++ ++++++++ * again on its way down.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++void btrfs_stop_parked_rbios(struct btrfs_fs_info *fs_info)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_stripe_hash_table *table = fs_info->stripe_hash_table;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!table || !fs_info->rmw_workers)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ for (int i = 0; i < 3 && !list_empty_careful(&table->parked); i++) {
+++++++++++++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, 0, U64_MAX, true);
+++++++++++++ ++ ++++++++ flush_workqueue(fs_info->rmw_workers);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ cancel_delayed_work_sync(&table->parked_work);
+++++++++++++ ++ ++++++++ flush_workqueue(fs_info->rmw_workers);
+++++++++++++ ++ ++++++++ WARN_ON(!list_empty_careful(&table->parked));
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
/*
* remove all cached entries and free the hash table
* used by unmount
static void cache_rbio(struct btrfs_raid_bio *rbio)
{
struct btrfs_stripe_hash_table *table;
+++++++++++++++++ ++++++++ const unsigned int limit = rbio_cache_limit(rbio->bioc->fs_info);
+++++++++++++ ++ ++++++++ bool pin;
if (!test_bit(RBIO_CACHE_READY_BIT, &rbio->flags))
return;
--
+++++++++++++ ++ ++++++++ pin = rbio_needs_pin(rbio);
+++++++++++++++++ ++++++++ /* Even a disabled cache keeps a stripe its run has not finished. */
+++++++++++++++++ ++++++++ if (!limit && !pin)
+++++++++++++++++ ++++++++ return;
++
table = rbio->bioc->fs_info->stripe_hash_table;
spin_lock(&table->cache_lock);
spin_unlock(&rbio->bio_list_lock);
------------- -- -------- if (table->cache_size > RBIO_CACHE_SIZE) {
------------- -- -------- struct btrfs_raid_bio *found;
------------- -- --------
------------- -- -------- found = list_last_entry(&table->stripe_cache,
------------- -- -------- struct btrfs_raid_bio,
------------- -- -------- stripe_cache);
------------- -- --------
------------- -- -------- if (found != rbio)
------------- -- -------- __remove_rbio_from_cache(found);
------------- -- -------- }
-- rbio_cache_prune(table, rbio, RBIO_CACHE_SIZE);
+++++++++++++++++ ++++++++ /* Prunes as far as needed: the limit can have been lowered. */
+++++++++++++++++ ++++++++ rbio_cache_prune(table, rbio, limit);
spin_unlock(&table->cache_lock);
}
}
}
--------- --- -- --------static void raid_wait_read_end_io(struct bio *bio)
--------- --- -- --------{
--------- --- -- -------- struct btrfs_raid_bio *rbio = bio->bi_private;
+++++++++ ++++++++++++++++/*
+++++++++ ++++++++++++++++ * Begin a submission round of @nr bios whose end_io functions report through
+++++++++ ++++++++++++++++ * rbio_dec_io(). stripes_pending gets one extra count on behalf of the
+++++++++ ++++++++++++++++ * submitter, dropped by rbio_wait_io(): this guarantees exactly one
+++++++++ ++++++++++++++++ * completion per round even when @nr is 0 or every bio finishes before the
+++++++++ ++++++++++++++++ * submitter starts waiting.
+++++++++ ++++++++++++++++ *
+++++++++ ++++++++++++++++ * The completion is what makes the round safe against a use-after-free the
+++++++++ ++++++++++++++++ * old bare waitqueue had: once the final bio's decrement made
+++++++++ ++++++++++++++++ * stripes_pending visible as zero, the waiter could pass its wait_event()
+++++++++ ++++++++++++++++ * condition check and free the rbio while the bio's end_io context was still
+++++++++ ++++++++++++++++ * inside wake_up() on the rbio's embedded waitqueue lock. With a
+++++++++ ++++++++++++++++ * completion, the waiter cannot return before complete() has released the
+++++++++ ++++++++++++++++ * completion's lock, so the completing context is finished with the rbio.
+++++++++ ++++++++++++++++ */
+++++++++ ++++++++++++++++static void rbio_begin_io(struct btrfs_raid_bio *rbio, unsigned int nr)
+++++++++ ++++++++++++++++{
+++++++++ ++++++++++++++++ reinit_completion(&rbio->io_done);
+++++++++ ++++++++++++++++ atomic_set(&rbio->stripes_pending, nr + 1);
+++++++++ ++++++++++++++++}
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++static void rbio_dec_io(struct btrfs_raid_bio *rbio)
+++++++++ ++++++++++++++++{
+++++++++ ++++++++++++++++ if (atomic_dec_and_test(&rbio->stripes_pending))
+++++++++ ++++++++++++++++ complete(&rbio->io_done);
+++++++++ ++++++++++++++++}
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++static void rbio_wait_io(struct btrfs_raid_bio *rbio)
+++++++++ ++++++++++++++++{
+++++++++ ++++++++++++++++ /* Drop the submitter's count from rbio_begin_io(). */
+++++++++ ++++++++++++++++ rbio_dec_io(rbio);
+++++++++ ++++++++++++++++ wait_for_completion(&rbio->io_done);
+++++++++ ++++++++++++++++}
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++/*
+++++++++ ++++++++++++++++ * Count a failed bio in the stats of the device it went to, as bio.c does for
+++++++++ ++++++++++++++++ * the other profiles. Look the device up by its bdev: a write to the replace
+++++++++ ++++++++++++++++ * target uses the source's pages, so the pages do not tell the device.
+++++++++ ++++++++++++++++ */
+++++++++ ++++++++++++++++static void rbio_log_dev_io_error(struct btrfs_raid_bio *rbio, struct bio *bio)
+++++++++ ++++++++++++++++{
+++++++++ ++++++++++++++++ struct btrfs_io_context *bioc = rbio->bioc;
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ for (int i = 0; i < bioc->num_stripes; i++) {
+++++++++ ++++++++++++++++ if (bioc->stripes[i].dev->bdev == bio->bi_bdev) {
+++++++++ ++++++++++++++++ btrfs_log_dev_io_error(bio, bioc->stripes[i].dev);
+++++++++ ++++++++++++++++ return;
+++++++++ ++++++++++++++++ }
+++++++++ ++++++++++++++++ }
+++++++++ ++++++++++++++++}
+++++++++ ++++++++++++++++
+++++++++ +++ ++ ++++++++static void raid_wait_read_end_io(struct bio *bio)
+++++++++ +++ ++ ++++++++{
+++++++++ +++ ++ ++++++++ struct btrfs_raid_bio *rbio = bio->bi_private;
if (bio->bi_status) {
+++++++++ ++++++++++++++++ rbio_log_dev_io_error(rbio, bio);
rbio_update_error_bitmap(rbio, bio);
} else {
set_bio_pages_uptodate(rbio, bio);
}
}
----------- - -- --------/*
----------- - -- -------- * To determine if we need to read any sector from the disk.
----------- - -- -------- * Should only be utilized in RMW path, to skip cached rbio.
----------- - -- -------- */
----------- - -- --------static bool need_read_stripe_sectors(struct btrfs_raid_bio *rbio)
+++++++++++++ ++ ++++++++static void rbio_record_pad(struct btrfs_raid_bio *rbio,
+++++++++++++ ++ ++++++++ const struct btrfs_stripe_pad_info *info)
+ {
----------- - -- -------- const unsigned int nr_data_sectors = rbio_nr_data_sectors(rbio);
----------- - -- -------- int i;
+++++++++++++ ++ ++++++++ rbio->pad_reason = info->reason;
+++++++++++++ ++ ++++++++ rbio->pad_norun = info->norun;
+++++++++++++ ++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
+++++++++++++ ++ ++++++++ rbio->pad_run_open = info->run_open;
+++++++++++++ ++ ++++++++ rbio->pad_run_class = info->run_class;
+++++++++++++ ++ ++++++++ rbio->pad_run_start = info->run_start;
+++++++++++++ ++ ++++++++ rbio->pad_run_end = info->run_end;
+++++++++++++ ++ ++++++++ rbio->pad_run_offset = info->run_offset;
+++++++++++++ ++ ++++++++#endif
+++++++++++++ ++ ++++++++}
+
----------- - -- -------- for (i = 0; i < nr_data_sectors; i++) {
----------- - -- -------- phys_addr_t paddr = rbio->stripe_paddrs[i * rbio->sector_nsteps];
+++++++++++++ ++ ++++++++static_assert(BTRFS_STRIPE_PAD_NR_REASONS + 1 ==
+++++++++++++ ++ ++++++++ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_park_stats.data_rmw_by));
+++++++++++++ ++ ++++++++static_assert(BTRFS_STRIPE_NORUN_NR ==
+++++++++++++ ++ ++++++++ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_park_stats.data_rmw_norun));
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
+++++++++++++ ++ ++++++++static_assert(BTRFS_STRIPE_PAD_NR_REASONS + 1 ==
+++++++++++++ ++ ++++++++ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_rmw_audit_stats.hole_by));
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/* What the audit found under one RMW's uncovered sectors. */
+++++++++++++ ++ ++++++++struct rmw_audit {
+++++++++++++ ++ ++++++++ int foreign; /* sectors under extents another root owns */
+++++++++++++ ++ ++++++++ int reloc; /* sectors under extents only relocation owns */
+++++++++++++ ++ ++++++++ u64 min_gen;
+++++++++++++ ++ ++++++++ u64 max_gen;
+++++++++++++ ++ ++++++++ u64 first; /* first foreign extent */
+++++++++++++ ++ ++++++++ u64 first_root; /* one other root referencing it (0: shared ref) */
+++++++++++++ ++ ++++++++};
+ +
--------- - - -- -------- /*
--------- - - -- -------- * We have a sector which doesn't have page nor uptodate,
--------- - - -- -------- * thus this rbio can not be cached one, as cached one must
--------- - - -- -------- * have all its data sectors present and uptodate.
--------- - - -- -------- */
--------- - - -- -------- if (paddr == INVALID_PADDR ||
--------- - - -- -------- !test_bit(i, rbio->stripe_uptodate_bitmap))
--------- - - -- -------- return true;
+++++++++++++ ++ ++++++++/* One extent being classified: its uncovered sectors and its owners. */
+++++++++++++ ++ ++++++++struct rmw_audit_ext {
+++++++++++++ ++ ++++++++ u64 bytenr;
+++++++++++++ ++ ++++++++ u64 gen;
+++++++++++++ ++ ++++++++ int sectors;
+++++++++++++ ++ ++++++++ int reloc_refs;
+++++++++++++ ++ ++++++++ int other_refs; /* includes shared refs: owner unknown, so foreign */
+++++++++++++ ++ ++++++++ u64 other_root;
+++++++++++++ ++ ++++++++};
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++static void rmw_audit_ext_ref(struct rmw_audit_ext *e, u64 root)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ if (root == BTRFS_DATA_RELOC_TREE_OBJECTID) {
+++++++++++++ ++ ++++++++ e->reloc_refs++;
+++++++++++++ ++ ++++++++ } else {
+++++++++++++ ++ ++++++++ e->other_refs++;
+++++++++++++ ++ ++++++++ if (!e->other_root)
+++++++++++++ ++ ++++++++ e->other_root = root;
+ + }
--------- - - -- -------- return false;
+ + }
+ +
--------- - - -- --------static void rmw_rbio(struct btrfs_raid_bio *rbio)
+++++++++++++ ++ ++++++++static void rmw_audit_ext_done(struct rmw_audit *a, struct rmw_audit_ext *e)
+ + {
--------- - - -- -------- struct bio_list bio_list;
--------- - - -- -------- int sectornr;
--------- - - -- -------- int ret = 0;
--------- - - -- --------
--------- - - -- -------- /*
--------- - - -- -------- * Allocate the pages for parity first, as P/Q pages will always be
--------- - - -- -------- * needed for both full-stripe and sub-stripe writes.
--------- - - -- -------- */
--------- - - -- -------- ret = alloc_rbio_parity_pages(rbio);
--------- - - -- -------- if (ret < 0)
+++++++++++++ ++ ++++++++ if (!e->sectors)
+ + goto out;
+++++++++++++ ++ ++++++++ /* No ref seen at all: be conservative and call it foreign. */
+++++++++++++ ++ ++++++++ if (e->other_refs || !e->reloc_refs) {
+++++++++++++ ++ ++++++++ if (!a->foreign) {
+++++++++++++ ++ ++++++++ a->first = e->bytenr;
+++++++++++++ ++ ++++++++ a->first_root = e->other_root;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ a->foreign += e->sectors;
+++++++++++++ ++ ++++++++ a->min_gen = min(a->min_gen, e->gen);
+++++++++++++ ++ ++++++++ a->max_gen = max(a->max_gen, e->gen);
+++++++++++++ ++ ++++++++ } else {
+++++++++++++ ++ ++++++++ a->reloc += e->sectors;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++out:
+++++++++++++ ++ ++++++++ memset(e, 0, sizeof(*e));
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Classify the data sectors of this rbio's stripe that the write does NOT
+++++++++++++ ++ ++++++++ * cover by the COMMITTED extent under them and that extent's owners. Those
+++++++++++++ ++ ++++++++ * are the sectors a torn RMW can destroy after a crash. An extent owned only
+++++++++++++ ++ ++++++++ * by the data reloc tree is relocation's own orphaned copy, discarded by
+++++++++++++ ++ ++++++++ * orphan cleanup if a crash comes before its pointers move; any other owner
+++++++++++++ ++ ++++++++ * is a real write hole. Commit root, no locking, as in
+++++++++++++ ++ ++++++++ * rmw_block_referenced(). Returns -1 when the lookup cannot answer.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * The commit root is switched shortly before the super lands, so an extent
+++++++++++++ ++ ++++++++ * of the transaction committing right now can count early: this errs towards
+++++++++++++ ++ ++++++++ * reporting a hole, never towards hiding one.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static int rmw_committed_uncovered(struct btrfs_raid_bio *rbio,
+++++++++++++ ++ ++++++++ struct rmw_audit *a)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++++ ++ ++++++++ const u64 stripe_start = rbio->bioc->full_stripe_logical;
+++++++++++++ ++ ++++++++ const u64 stripe_end = stripe_start +
+++++++++++++ ++ ++++++++ (u64)rbio->nr_data * rbio->bioc->stripe_len;
+++++++++++++ ++ ++++++++ const u32 bits = fs_info->sectorsize_bits;
+++++++++++++ ++ ++++++++ struct btrfs_root *extent_root = btrfs_extent_root(fs_info, stripe_start);
+++++++++++++ ++ ++++++++ struct rmw_audit_ext e = { 0 };
+++++++++++++ ++ ++++++++ struct btrfs_path *path;
+++++++++++++ ++ ++++++++ struct btrfs_key key;
+++++++++++++ ++ ++++++++ int ret;
+ +
+++++++++++++ ++ ++++++++ memset(a, 0, sizeof(*a));
+++++++++++++ ++ ++++++++ a->min_gen = U64_MAX;
+ + /*
--------- - - -- -------- * Either full stripe write, or we have every data sector already
--------- - - -- -------- * cached, can go to write path immediately.
+++++++++++++ ++ ++++++++ * The caller holds the full stripe lock, so nothing here may wait for
+++++++++++++ ++ ++++++++ * something that can wait for this stripe: a tree block read that
+++++++++++++ ++ ++++++++ * needs a raid56 rebuild of it (mixed block groups keep tree blocks in
+++++++++++++ ++ ++++++++ * it, see fill_data_csums(), so they are not audited), or
+++++++++++++ ++ ++++++++ * commit_root_sem, whose holder may be reading such a block. Read
+++++++++++++ ++ ++++++++ * cached blocks only, and take the semaphore only if it is free. Each
+++++++++++++ ++ ++++++++ * miss counts as an audit error.
+ + */
--------- - - -- -------- if (!rbio_is_full(rbio) && need_read_stripe_sectors(rbio)) {
--------- - - -- -------- /*
--------- - - -- -------- * Now we're doing sub-stripe write, also need all data stripes
--------- - - -- -------- * to do the full RMW.
--------- - - -- -------- */
--------- - - -- -------- ret = alloc_rbio_data_pages(rbio);
--------- - - -- -------- if (ret < 0)
--------- - - -- -------- goto out;
--------- - - -- --------
--------- - - -- -------- index_rbio_pages(rbio);
+++++++++++++ ++ ++++++++ if (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_METADATA)
+++++++++++++ ++ ++++++++ return -1;
+++++++++++++ ++ ++++++++ path = btrfs_alloc_path();
+++++++++++++ ++ ++++++++ if (!path)
+++++++++++++ ++ ++++++++ return -1;
+++++++++++++ ++ ++++++++ path->search_commit_root = 1;
+++++++++++++ ++ ++++++++ path->skip_locking = 1;
+++++++++++++ ++ ++++++++ path->nowait = 1;
+ +
--------- - - -- -------- ret = rmw_read_wait_recover(rbio);
+++++++++++++ ++ ++++++++ if (!down_read_trylock(&fs_info->commit_root_sem)) {
+++++++++++++ ++ ++++++++ btrfs_free_path(path);
+++++++++++++ ++ ++++++++ return -1;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ key.objectid = stripe_start;
+++++++++++++ ++ ++++++++ key.type = BTRFS_EXTENT_ITEM_KEY;
+++++++++++++ ++ ++++++++ key.offset = 0;
+++++++++++++ ++ ++++++++ ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+++++++++++++ ++ ++++++++ if (ret < 0)
+++++++++++++ ++ ++++++++ goto fail;
+++++++++++++ ++ ++++++++ /* An extent starting below the stripe may reach into it. */
+++++++++++++ ++ ++++++++ ret = btrfs_previous_extent_item(extent_root, path,
+++++++++++++ ++ ++++++++ stripe_start > SZ_1G ? stripe_start - SZ_1G : 0);
+++++++++++++ ++ ++++++++ if (ret < 0)
+++++++++++++ ++ ++++++++ goto fail;
+++++++++++++ ++ ++++++++ if (ret > 0) {
+++++++++++++ ++ ++++++++ btrfs_release_path(path);
+++++++++++++ ++ ++++++++ ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+ + if (ret < 0)
--------- - - -- -------- goto out;
+++++++++++++ ++ ++++++++ goto fail;
+ + }
+++++++++++++ ++ ++++++++ while (1) {
+++++++++++++ ++ ++++++++ struct extent_buffer *leaf = path->nodes[0];
+++++++++++++ ++ ++++++++ const int slot = path->slots[0];
+++++++++++++ ++ ++++++++ struct btrfs_extent_item *ei;
+++++++++++++ ++ ++++++++ unsigned long ptr, end;
+++++++++++++ ++ ++++++++ u64 lo, hi;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (slot >= btrfs_header_nritems(leaf)) {
+++++++++++++ ++ ++++++++ ret = btrfs_next_leaf(extent_root, path);
+++++++++++++ ++ ++++++++ if (ret < 0)
+++++++++++++ ++ ++++++++ goto fail;
+++++++++++++ ++ ++++++++ if (ret > 0)
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ btrfs_item_key_to_cpu(leaf, &key, slot);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Keyed backrefs of the extent being classified. */
+++++++++++++ ++ ++++++++ if (e.sectors && key.objectid == e.bytenr) {
+++++++++++++ ++ ++++++++ if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
+++++++++++++ ++ ++++++++ struct btrfs_extent_data_ref *dref;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ dref = btrfs_item_ptr(leaf, slot,
+++++++++++++ ++ ++++++++ struct btrfs_extent_data_ref);
+++++++++++++ ++ ++++++++ rmw_audit_ext_ref(&e, btrfs_extent_data_ref_root(leaf, dref));
+++++++++++++ ++ ++++++++ } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
+++++++++++++ ++ ++++++++ e.other_refs++;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ path->slots[0]++;
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (key.type != BTRFS_EXTENT_ITEM_KEY) {
+++++++++++++ ++ ++++++++ if (key.objectid >= stripe_end)
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ path->slots[0]++;
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ rmw_audit_ext_done(a, &e);
+++++++++++++ ++ ++++++++ if (key.objectid >= stripe_end)
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ if (key.objectid + key.offset <= stripe_start) {
+++++++++++++ ++ ++++++++ path->slots[0]++;
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item);
+++++++++++++ ++ ++++++++ lo = max(key.objectid, stripe_start);
+++++++++++++ ++ ++++++++ hi = min(key.objectid + key.offset, stripe_end);
+++++++++++++ ++ ++++++++ for (u64 b = lo; b < hi; b += fs_info->sectorsize)
+++++++++++++ ++ ++++++++ if (rbio->bio_paddrs[((b - stripe_start) >> bits) * rbio->sector_nsteps] == INVALID_PADDR)
+++++++++++++ ++ ++++++++ e.sectors++;
+++++++++++++ ++ ++++++++ if (!e.sectors) {
+++++++++++++ ++ ++++++++ path->slots[0]++;
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ e.bytenr = key.objectid;
+++++++++++++ ++ ++++++++ e.gen = btrfs_extent_generation(leaf, ei);
+++++++++++++ ++ ++++++++ /* Inline backrefs: data extents carry no tree_block_info. */
+++++++++++++ ++ ++++++++ ptr = (unsigned long)(ei + 1);
+++++++++++++ ++ ++++++++ end = (unsigned long)ei + btrfs_item_size(leaf, slot);
+++++++++++++ ++ ++++++++ while (ptr < end) {
+++++++++++++ ++ ++++++++ struct btrfs_extent_inline_ref *iref = (void *)ptr;
+++++++++++++ ++ ++++++++ const int type = btrfs_extent_inline_ref_type(leaf, iref);
+++++++++++++ ++ ++++++++ const u32 size = btrfs_extent_inline_ref_size(type);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (type == BTRFS_EXTENT_DATA_REF_KEY) {
+++++++++++++ ++ ++++++++ struct btrfs_extent_data_ref *dref =
+++++++++++++ ++ ++++++++ (void *)&iref->offset;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ rmw_audit_ext_ref(&e, btrfs_extent_data_ref_root(leaf, dref));
+++++++++++++ ++ ++++++++ } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
+++++++++++++ ++ ++++++++ e.other_refs++;
+++++++++++++ ++ ++++++++ } else if (type != BTRFS_EXTENT_OWNER_REF_KEY) {
+++++++++++++ ++ ++++++++ /* Not a data ref we understand: foreign. */
+++++++++++++ ++ ++++++++ e.other_refs++;
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (!size)
+++++++++++++ ++ ++++++++ break;
+++++++++++++ ++ ++++++++ ptr += size;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ path->slots[0]++;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ rmw_audit_ext_done(a, &e);
+++++++++++++ ++ ++++++++ up_read(&fs_info->commit_root_sem);
+++++++++++++ ++ ++++++++ btrfs_free_path(path);
+++++++++++++ ++ ++++++++ return a->foreign + a->reloc;
+++++++++++++ ++ ++++++++fail:
+++++++++++++ ++ ++++++++ up_read(&fs_info->commit_root_sem);
+++++++++++++ ++ ++++++++ btrfs_free_path(path);
+++++++++++++ ++ ++++++++ return -1;
+++++++++++++ ++ ++++++++}
+ +
--------- - - -- -------- /*
--------- - - -- -------- * At this stage we're not allowed to add any new bios to the
--------- - - -- -------- * bio list any more, anyone else that wants to change this stripe
--------- - - -- -------- * needs to do their own rmw.
--------- - - -- -------- */
--------- - - -- -------- spin_lock(&rbio->bio_list_lock);
--------- - - -- -------- set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
--------- - - -- -------- spin_unlock(&rbio->bio_list_lock);
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * With /sys/fs/btrfs/<uuid>/debug/stripe_rmw_audit set: prove from the
+++++++++++++ ++ ++++++++ * committed extent tree, not from the allocator's design, that a data RMW put
+++++++++++++ ++ ++++++++ * no other owner's committed data at risk, and log the writer of any that
+++++++++++++ ++ ++++++++ * did (and of every NO_RUN RMW, the class the allocator cannot explain by
+++++++++++++ ++ ++++++++ * construction). @idx is the RMW's data_rmw_by index.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static void rbio_audit_rmw(struct btrfs_raid_bio *rbio, u8 idx)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++++ ++ ++++++++ const u64 stripe_start = rbio->bioc->full_stripe_logical;
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ struct rmw_audit a;
+++++++++++++ ++ ++++++++ u64 root_id = 0, ino = 0, foff = 0, oflags = 0, oseq = 0, odisk = 0;
+++++++++++++ ++ ++++++++ u64 bg_flags = 0;
+++++++++++++ ++ ++++++++ bool orun = false;
+++++++++++++ ++ ++++++++ int nbios = 0;
+++++++++++++ ++ ++++++++ int ro = -1;
+++++++++++++ ++ ++++++++ int ret;
+++++++++++++ ++ ++++++++ struct bio *bio;
+ +
--------- - - -- -------- bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors);
+++++++++++++ ++ ++++++++ if (!READ_ONCE(fs_info->stripe_rmw_audit))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.audited);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ spin_lock(&rbio->bio_list_lock);
+++++++++++++ ++ ++++++++ bio_list_for_each(bio, &rbio->bio_list) {
+++++++++++++ ++ ++++++++ struct btrfs_bio *bbio = btrfs_bio(bio);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (nbios++)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (!bbio->inode ||
+++++++++++++ ++ ++++++++ btrfs_ino(bbio->inode) == BTRFS_BTREE_INODE_OBJECTID)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ root_id = btrfs_root_id(bbio->inode->root);
+++++++++++++ ++ ++++++++ ino = btrfs_ino(bbio->inode);
+++++++++++++ ++ ++++++++ foff = bbio->file_offset;
+++++++++++++ ++ ++++++++ if (bbio->ordered) {
+++++++++++++ ++ ++++++++ oflags = bbio->ordered->flags;
+++++++++++++ ++ ++++++++ oseq = bbio->ordered->stripe_seq;
+++++++++++++ ++ ++++++++ odisk = bbio->ordered->disk_bytenr;
+++++++++++++ ++ ++++++++ orun = bbio->ordered->stripe_run != NULL;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ spin_unlock(&rbio->bio_list_lock);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* The audit needs bio sectors mapped; idempotent. */
+++++++++++++ ++ ++++++++ index_rbio_pages(rbio);
+++++++++++++ ++ ++++++++ ret = rmw_committed_uncovered(rbio, &a);
+++++++++++++ ++ ++++++++ if (ret < 0) {
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.audit_err);
+++++++++++++ ++ ++++++++ } else if (a.foreign) {
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.hole_by[idx]);
+++++++++++++ ++ ++++++++ atomic64_add(a.foreign, &fs_info->stripe_rmw_audit_stats.hole_sectors);
+++++++++++++ ++ ++++++++ if (root_id == BTRFS_DATA_RELOC_TREE_OBJECTID)
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.reloc_writer_foreign);
+++++++++++++ ++ ++++++++ } else if (a.reloc) {
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.hole_reloc_only);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (ret > 0)
+++++++++++++ ++ ++++++++ atomic64_add(a.reloc, &fs_info->stripe_rmw_audit_stats.hole_reloc_sectors);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (idx == BTRFS_STRIPE_PAD_NO_RUN) {
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+++++++++++++ ++ ++++++++ if (bg) {
+++++++++++++ ++ ++++++++ ro = data_race(bg->ro);
+++++++++++++ ++ ++++++++ bg_flags = bg->flags;
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Separate call sites, so separate rate limits: a flood of NO_RUN
+++++++++++++ ++ ++++++++ * lines cannot starve the hole lines.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (ret > 0 && a.foreign)
+++++++++++++ ++ ++++++++ btrfs_warn_rl(fs_info,
+++++++++++++ ++ ++++++++"rmw audit: write hole: stripe %llu reason %u norun %u foreign %d reloc %d gen %llu..%llu first %llu first_root %lld fsgen %llu committed %llu writer root %lld ino %llu foff %llu oe_flags 0x%llx oe_disk %llu oe_seq %llu oe_run %d nbios %d",
+++++++++++++ ++ ++++++++ stripe_start, idx, rbio->pad_norun, a.foreign,
+++++++++++++ ++ ++++++++ a.reloc, a.min_gen, a.max_gen, a.first,
+++++++++++++ ++ ++++++++ (s64)a.first_root, btrfs_get_fs_generation(fs_info),
+++++++++++++ ++ ++++++++ btrfs_get_last_trans_committed(fs_info),
+++++++++++++ ++ ++++++++ (s64)root_id, ino, foff, oflags, odisk, oseq, orun,
+++++++++++++ ++ ++++++++ nbios);
+++++++++++++ ++ ++++++++ else if (idx == BTRFS_STRIPE_PAD_NO_RUN)
+++++++++++++ ++ ++++++++ btrfs_info_rl(fs_info,
+++++++++++++ ++ ++++++++"rmw audit: no_run RMW stripe %llu norun %u bg_ro %d bg_flags 0x%llx run %llu..%llu off %llu open %u class %u reloc_sectors %d fsgen %llu committed %llu writer root %lld ino %llu foff %llu oe_flags 0x%llx oe_seq %llu oe_run %d nbios %d",
+++++++++++++ ++ ++++++++ stripe_start, rbio->pad_norun, ro, bg_flags,
+++++++++++++ ++ ++++++++ rbio->pad_run_start, rbio->pad_run_end,
+++++++++++++ ++ ++++++++ rbio->pad_run_offset, rbio->pad_run_open,
+++++++++++++ ++ ++++++++ rbio->pad_run_class, ret > 0 ? a.reloc : 0,
+++++++++++++ ++ ++++++++ btrfs_get_fs_generation(fs_info),
+++++++++++++ ++ ++++++++ btrfs_get_last_trans_committed(fs_info),
+++++++++++++ ++ ++++++++ (s64)root_id, ino, foff, oflags, oseq, orun, nbios);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++#else
+++++++++++++ ++ ++++++++static void rbio_audit_rmw(struct btrfs_raid_bio *rbio, u8 idx) { }
+++++++++++++ ++ ++++++++#endif
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Attribute a read-modify-write to why padding declined for this rbio. With
+++++++++++++ ++ ++++++++ * allow_rmw empty, a data RMW in a stripe_alloc group is expected in two
+++++++++++++ ++ ++++++++ * places only: at the open run's own frontier (UNCOVERED: the sectors it
+++++++++++++ ++ ++++++++ * reads are the same run's, in the same transaction, not yet durable), and
+++++++++++++ ++ ++++++++ * from relocation writing into its own preallocated extents, which nothing
+++++++++++++ ++ ++++++++ * but the data reloc inode references until relocation moves the pointers
+++++++++++++ ++ ++++++++ * after writeback. Any other data RMW in a covered group is unexplained.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static void rbio_count_rmw_reason(struct btrfs_raid_bio *rbio)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++++ ++ ++++++++ u8 idx = rbio->pad_reason;
+++++++++++++ ++ ++++++++ bool reloc_writer = false;
+++++++++++++ ++ ++++++++ struct bio *bio;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (idx >= BTRFS_STRIPE_PAD_NR_REASONS)
+++++++++++++ ++ ++++++++ idx = BTRFS_STRIPE_PAD_NR_REASONS; /* not tried */
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_by[idx]);
+++++++++++++ ++ ++++++++ if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (idx == BTRFS_STRIPE_PAD_NO_RUN) {
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (rbio->pad_norun < BTRFS_STRIPE_NORUN_NR)
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_norun[rbio->pad_norun]);
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info,
+++++++++++++ ++ ++++++++ rbio->bioc->full_stripe_logical);
+++++++++++++ ++ ++++++++ if (bg) {
+++++++++++++ ++ ++++++++ if (data_race(bg->ro))
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_norun_ro);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* The writer: the first bio's inode. */
+++++++++++++ ++ ++++++++ spin_lock(&rbio->bio_list_lock);
+++++++++++++ ++ ++++++++ bio = bio_list_peek(&rbio->bio_list);
+++++++++++++ ++ ++++++++ if (bio && btrfs_bio(bio)->inode)
+++++++++++++ ++ ++++++++ reloc_writer = btrfs_is_data_reloc_root(btrfs_bio(bio)->inode->root);
+++++++++++++ ++ ++++++++ spin_unlock(&rbio->bio_list_lock);
+++++++++++++ ++ ++++++++ /* Disjoint from the frontier count, so the two can be summed. */
+++++++++++++ ++ ++++++++ if (reloc_writer && idx != BTRFS_STRIPE_PAD_UNCOVERED)
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_reloc_writer);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ rbio_audit_rmw(rbio, idx);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Try to avoid the RMW read phase of a sub-stripe write by zero-filling the
+++++++++++++ ++ ++++++++ * uncovered data sectors. Legal only where the stripe's run says a sector
+++++++++++++ ++ ++++++++ * is dead: never allocated, handed back, or allocated with its write still
+++++++++++++ ++ ++++++++ * to come. That write, like any allocation the run makes after this check,
+++++++++++++ ++ ++++++++ * serializes behind this rbio's stripe lock and lands over the zeros.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static bool rmw_try_pad_full(struct btrfs_raid_bio *rbio)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++++ ++ ++++++++ const u64 stripe_start = rbio->bioc->full_stripe_logical;
+++++++++++++ ++ ++++++++ const u64 stripe_len = (u64)rbio->nr_data * rbio->bioc->stripe_len;
+++++++++++++ ++ ++++++++ const u32 sectorsize = fs_info->sectorsize;
+++++++++++++ ++ ++++++++ const u32 nsectors = rbio->nr_data * rbio->stripe_nsectors;
+++++++++++++ ++ ++++++++ struct btrfs_stripe_pad_info info;
+++++++++++++ ++ ++++++++ unsigned long *dead;
+++++++++++++ ++ ++++++++ unsigned long *unwritten;
+++++++++++++ ++ ++++++++ u32 covered = 0;
+++++++++++++ ++ ++++++++ u32 nr_unwritten = 0;
+++++++++++++ ++ ++++++++ bool ok = false;
+++++++++++++ ++ ++++++++ int i;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ rbio->pad_reason = RBIO_PAD_NOT_TRIED;
+++++++++++++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ dead = bitmap_zalloc(nsectors, GFP_NOFS);
+++++++++++++ ++ ++++++++ unwritten = bitmap_zalloc(nsectors, GFP_NOFS);
+++++++++++++ ++ ++++++++ if (!dead || !unwritten) {
+++++++++++++ ++ ++++++++ bitmap_free(dead);
+++++++++++++ ++ ++++++++ bitmap_free(unwritten);
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (!btrfs_stripe_run_pad_mask(fs_info, stripe_start, stripe_len, dead,
+++++++++++++ ++ ++++++++ unwritten, &info)) {
+++++++++++++ ++ ++++++++ rbio_record_pad(rbio, &info);
+++++++++++++ ++ ++++++++ trace_btrfs_stripe_pad_decline(fs_info, stripe_start,
+++++++++++++ ++ ++++++++ stripe_len, 0, 0, nsectors,
+++++++++++++ ++ ++++++++ info.run_start, info.run_end,
+++++++++++++ ++ ++++++++ info.run_offset,
+++++++++++++ ++ ++++++++ info.inflight_bytes,
+++++++++++++ ++ ++++++++ info.reason);
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Populate bio_paddrs so covered sectors are visible below. */
+++++++++++++ ++ ++++++++ index_rbio_pages(rbio);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ for (i = 0; i < nsectors; i++)
+++++++++++++ ++ ++++++++ if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
+++++++++++++ ++ ++++++++ covered++;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Every uncovered sector must be dead per the run. A live uncovered
+++++++++++++ ++ ++++++++ * sector that was already written holds data this write does not
+++++++++++++ ++ ++++++++ * carry, so refuse (UNCOVERED); the stripe pin (rbio_needs_pin())
+++++++++++++ ++ ++++++++ * normally supplies such sectors from the cache, so nothing is read.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ for (i = 0; i < nsectors; i++) {
+++++++++++++ ++ ++++++++ if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (!test_bit(i, dead)) {
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_uncovered);
+++++++++++++ ++ ++++++++ info.reason = BTRFS_STRIPE_PAD_UNCOVERED;
+++++++++++++ ++ ++++++++ rbio_record_pad(rbio, &info);
+++++++++++++ ++ ++++++++ trace_btrfs_stripe_pad_decline(fs_info, stripe_start,
+++++++++++++ ++ ++++++++ stripe_len, 0, covered,
+++++++++++++ ++ ++++++++ nsectors,
+++++++++++++ ++ ++++++++ info.run_start,
+++++++++++++ ++ ++++++++ info.run_end,
+++++++++++++ ++ ++++++++ info.run_offset,
+++++++++++++ ++ ++++++++ info.inflight_bytes,
+++++++++++++ ++ ++++++++ info.reason);
+++++++++++++ ++ ++++++++ goto out;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (test_bit(i, unwritten))
+++++++++++++ ++ ++++++++ nr_unwritten++;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ ok = true;
+++++++++++++ ++ ++++++++out:
+++++++++++++ ++ ++++++++ bitmap_free(dead);
+++++++++++++ ++ ++++++++ bitmap_free(unwritten);
+++++++++++++ ++ ++++++++ if (!ok)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* The oracle agreed: only a page allocation failure refuses now. */
+++++++++++++ ++ ++++++++ rbio->pad_reason = BTRFS_STRIPE_PAD_OK;
+++++++++++++ ++ ++++++++ if (alloc_rbio_data_pages(rbio) < 0)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) {
+++++++++++++ ++ ++++++++ const int stripe_nr = i / rbio->stripe_nsectors;
+++++++++++++ ++ ++++++++ const int sectornr = i % rbio->stripe_nsectors;
+++++++++++++ ++ ++++++++ int step;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ for (step = 0; step < rbio->sector_nsteps; step++) {
+++++++++++++ ++ ++++++++ phys_addr_t paddr = rbio_stripe_paddr(rbio, stripe_nr,
+++++++++++++ ++ ++++++++ sectornr, step);
+++++++++++++ ++ ++++++++ void *kaddr = kmap_local_paddr(paddr);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ memset(kaddr, 0, min_t(u32, sectorsize, PAGE_SIZE));
+++++++++++++ ++ ++++++++ kunmap_local(kaddr);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ set_bit(i, rbio->stripe_uptodate_bitmap);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ set_bit(RBIO_PADDED_BIT, &rbio->flags);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * The whole stripe goes out: every vertical position is written.
+++++++++++++ ++ ++++++++ * Under bio_list_lock: until RBIO_RMW_LOCKED_BIT is set, merge_rbio()
+++++++++++++ ++ ++++++++ * can still OR another rbio's bits into this bitmap.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ spin_lock(&rbio->bio_list_lock);
+++++++++++++ ++ ++++++++ bitmap_set(&rbio->dbitmap, 0, rbio->stripe_nsectors);
+++++++++++++ ++ ++++++++ spin_unlock(&rbio->bio_list_lock);
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.padded);
+++++++++++++ ++ ++++++++ if (nr_unwritten)
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.padded_unwritten);
+++++++++++++ ++ ++++++++ return true;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++ + ++ ++++++++/*
+++++++++++ + ++ ++++++++ * To determine if we need to read any sector from the disk.
+++++++++++ + ++ ++++++++ * Should only be utilized in RMW path, to skip cached rbio.
+++++++++++ + ++ ++++++++ */
+++++++++++ + ++ ++++++++static bool need_read_stripe_sectors(struct btrfs_raid_bio *rbio)
+++++++++++ + ++ ++++++++{
+++++++++++ + ++ ++++++++ const unsigned int nr_data_sectors = rbio_nr_data_sectors(rbio);
+++++++++++ + ++ ++++++++ int i;
+++++++++++ + ++ ++++++++
+++++++++++ + ++ ++++++++ for (i = 0; i < nr_data_sectors; i++) {
+++++++++++ + ++ ++++++++ phys_addr_t paddr = rbio->stripe_paddrs[i * rbio->sector_nsteps];
+++++++++ + + ++ ++++++++
+++++++++ + + ++ ++++++++ /*
+++++++++ + + ++ ++++++++ * We have a sector which doesn't have page nor uptodate,
+++++++++ + + ++ ++++++++ * thus this rbio can not be cached one, as cached one must
+++++++++ + + ++ ++++++++ * have all its data sectors present and uptodate.
+++++++++ + + ++ ++++++++ */
+++++++++ + + ++ ++++++++ if (paddr == INVALID_PADDR ||
+++++++++ + + ++ ++++++++ !test_bit(i, rbio->stripe_uptodate_bitmap))
+++++++++ + + ++ ++++++++ return true;
+++++++++ + + ++ ++++++++ }
+++++++++ + + ++ ++++++++ return false;
+++++++++ + + ++ ++++++++}
+++++++++ + + ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Read the header of the tree block at data-stripe sector @i, if there is
+++++++++++++ ++ ++++++++ * one, from whichever copy of the sector this rbio is holding.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * @from_bio picks the source: the caller's pages (what this write is putting
+++++++++++++ ++ ++++++++ * there) or the stripe pages (what an RMW read found on the disk). Returns
+++++++++++++ ++ ++++++++ * false when the sector holds no tree block that belongs at this address --
+++++++++++++ ++ ++++++++ * free space, or the remains of one from a previous life of the stripe.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static bool rmw_peek_eb_generation(struct btrfs_raid_bio *rbio, u32 i,
+++++++++++++ ++ ++++++++ bool from_bio, u64 *generation)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++++ ++ ++++++++ const int stripe_nr = i / rbio->stripe_nsectors;
+++++++++++++ ++ ++++++++ const int sectornr = i % rbio->stripe_nsectors;
+++++++++++++ ++ ++++++++ const u64 logical = rbio->bioc->full_stripe_logical +
+++++++++++++ ++ ++++++++ (u64)stripe_nr * rbio->bioc->stripe_len +
+++++++++++++ ++ ++++++++ (u64)sectornr * fs_info->sectorsize;
+++++++++++++ ++ ++++++++ const phys_addr_t *paddrs = from_bio ? rbio->bio_paddrs
+++++++++++++ ++ ++++++++ : rbio->stripe_paddrs;
+++++++++++++ ++ ++++++++ struct btrfs_header *hdr;
+++++++++++++ ++ ++++++++ phys_addr_t paddr;
+++++++++++++ ++ ++++++++ u64 found_bytenr;
+++++++++++++ ++ ++++++++ bool bad_fsid;
+++++++++++++ ++ ++++++++ void *kaddr;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ paddr = paddrs[i * rbio->sector_nsteps];
+++++++++++++ ++ ++++++++ if (paddr == INVALID_PADDR)
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++ if (!from_bio && !test_bit(i, rbio->stripe_uptodate_bitmap))
+++++++++++++ ++ ++++++++ return false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ kaddr = kmap_local_paddr(paddr);
+++++++++++++ ++ ++++++++ hdr = kaddr;
+++++++++++++ ++ ++++++++ found_bytenr = btrfs_stack_header_bytenr(hdr);
+++++++++++++ ++ ++++++++ *generation = btrfs_stack_header_generation(hdr);
+++++++++++++ ++ ++++++++ bad_fsid = memcmp(hdr->fsid, fs_info->fs_devices->metadata_uuid,
+++++++++++++ ++ ++++++++ BTRFS_FSID_SIZE) != 0;
+++++++++++++ ++ ++++++++ kunmap_local(kaddr);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ return found_bytenr == logical && !bad_fsid;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Is @bytenr a tree block the committed extent tree still references? Read
+++++++++++++ ++ ++++++++ * the commit root without locking, as the caching thread does, so this is
+++++++++++++ ++ ++++++++ * safe from the raid56 worker with no transaction to join.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Returns 1 if it is (or the lookup failed), 0 if not, and -EAGAIN if the
+++++++++++++ ++ ++++++++ * answer needs a tree block that is not in memory. The caller holds the
+++++++++++++ ++ ++++++++ * full stripe lock, so it must not read from disk: a read of a block in
+++++++++++++ ++ ++++++++ * this stripe that needs reconstruction would wait for that lock.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static int rmw_block_referenced(struct btrfs_fs_info *fs_info, u64 bytenr,
+++++++++++++ ++ ++++++++ u64 header_gen, u64 *refs, u64 *extent_gen)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_root *extent_root = btrfs_extent_root(fs_info, bytenr);
+++++++++++++ ++ ++++++++ struct btrfs_path *path;
+++++++++++++ ++ ++++++++ struct btrfs_key key;
+++++++++++++ ++ ++++++++ int live = 1;
+++++++++++++ ++ ++++++++ int ret;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ *refs = 0;
+++++++++++++ ++ ++++++++ *extent_gen = 0;
+++++++++++++ ++ ++++++++ path = btrfs_alloc_path();
+++++++++++++ ++ ++++++++ if (!path)
+++++++++++++ ++ ++++++++ return 1;
+++++++++++++ ++ ++++++++ path->search_commit_root = 1;
+++++++++++++ ++ ++++++++ path->skip_locking = 1;
+++++++++++++ ++ ++++++++ path->nowait = 1;
+++++++++++++ ++ ++++++++ key.objectid = bytenr;
+++++++++++++ ++ ++++++++ key.type = BTRFS_METADATA_ITEM_KEY;
+++++++++++++ ++ ++++++++ key.offset = (u64)-1;
+++++++++++++ ++ ++++++++ if (!down_read_trylock(&fs_info->commit_root_sem)) {
+++++++++++++ ++ ++++++++ btrfs_free_path(path);
+++++++++++++ ++ ++++++++ return -EAGAIN;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+++++++++++++ ++ ++++++++ if (ret > 0)
+++++++++++++ ++ ++++++++ ret = btrfs_previous_extent_item(extent_root, path, bytenr);
+++++++++++++ ++ ++++++++ if (ret == 0) {
+++++++++++++ ++ ++++++++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
+++++++++++++ ++ ++++++++ if (key.objectid == bytenr &&
+++++++++++++ ++ ++++++++ (key.type == BTRFS_METADATA_ITEM_KEY ||
+++++++++++++ ++ ++++++++ key.type == BTRFS_EXTENT_ITEM_KEY)) {
+++++++++++++ ++ ++++++++ struct btrfs_extent_item *ei;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
+++++++++++++ ++ ++++++++ struct btrfs_extent_item);
+++++++++++++ ++ ++++++++ *refs = btrfs_extent_refs(path->nodes[0], ei);
+++++++++++++ ++ ++++++++ *extent_gen = btrfs_extent_generation(path->nodes[0], ei);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * An extent item at this bytenr from a LATER
+++++++++++++ ++ ++++++++ * transaction than the header on disk belongs to a
+++++++++++++ ++ ++++++++ * new block allocated here whose write has not landed
+++++++++++++ ++ ++++++++ * yet; the header is the dead previous occupant.
+++++++++++++ ++ ++++++++ * Only an item of the header's own generation makes
+++++++++++++ ++ ++++++++ * the block on disk live.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ live = (*refs != 0 && *extent_gen == header_gen);
+++++++++++++ ++ ++++++++ } else {
+++++++++++++ ++ ++++++++ live = 0;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ } else if (ret > 0) {
+++++++++++++ ++ ++++++++ live = 0;
+++++++++++++ ++ ++++++++ } else if (ret == -EAGAIN) {
+++++++++++++ ++ ++++++++ live = -EAGAIN;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ up_read(&fs_info->commit_root_sem);
+++++++++++++ ++ ++++++++ btrfs_free_path(path);
+++++++++++++ ++ ++++++++ return live;
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Decide whether an uncovered metadata read-modify-write is a write hole or
+++++++++++++ ++ ++++++++ * only a cost, from the tree blocks it read: their parity is about to be
+++++++++++++ ++ ++++++++ * rewritten while their own sectors stay put.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * - Same generation as the blocks this write carries (meta_rmw_cur): the
+++++++++++++ ++ ++++++++ * block belongs to the running transaction, which a crash discards --
+++++++++++++ ++ ++++++++ * except a log tree block, which is durable once its fsync returns.
+++++++++++++ ++ ++++++++ * The audit does not tell log blocks apart, so it cannot see a write
+++++++++++++ ++ ++++++++ * hole over fsynced log blocks; keeping them out of shared stripes is
+++++++++++++ ++ ++++++++ * stripe_log_run's job.
+++++++++++++ ++ ++++++++ * - Older, and still referenced by the committed extent tree: a write hole
+++++++++++++ ++ ++++++++ * over committed metadata (meta_rmw_old).
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Comparing against this rbio's own blocks, not fs_info's committed
+++++++++++++ ++ ++++++++ * generation, keeps a concurrent commit from skewing the verdict.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static void audit_uncovered_rmw(struct btrfs_raid_bio *rbio)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++++ ++ ++++++++ const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits;
+++++++++++++ ++ ++++++++ const u32 nsectors = rbio->nr_data * rbio->stripe_nsectors;
+++++++++++++ ++ ++++++++ u64 write_gen = 0;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Tree blocks only: metadata and system groups. A mixed group's
+++++++++++++ ++ ++++++++ * sectors need not hold tree blocks.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ if (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ if (!(rbio->bioc->map_type & (BTRFS_BLOCK_GROUP_METADATA |
+++++++++++++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Both loops read bio_paddrs to tell what this write covers, and the
+++++++++++++ ++ ++++++++ * paths that reach here do not all populate it (a cached rbio that
+++++++++++++ ++ ++++++++ * failed to pad never indexes). Idempotent, so just do it.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ index_rbio_pages(rbio);
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* The generation this write is putting into the stripe. */
+++++++++++++ ++ ++++++++ for (u32 i = 0; i < nsectors; i += sectors_per_tree) {
+++++++++++++ ++ ++++++++ u64 gen;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (rmw_peek_eb_generation(rbio, i, true, &gen))
+++++++++++++ ++ ++++++++ write_gen = max(write_gen, gen);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (!write_gen)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ for (u32 i = 0; i < nsectors; i += sectors_per_tree) {
+++++++++++++ ++ ++++++++ u64 gen;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /* Covered by this write: it is what we are putting there. */
+++++++++++++ ++ ++++++++ if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ if (!rmw_peek_eb_generation(rbio, i, false, &gen)) {
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_free);
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (gen >= write_gen) {
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_cur);
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * An older header is a write hole only if the committed extent
+++++++++++++ ++ ++++++++ * tree still references the block: a reclaimed stripe keeps the
+++++++++++++ ++ ++++++++ * headers of dead blocks, and tearing those harms nothing a
+++++++++++++ ++ ++++++++ * reader can reach. The commit root is what a crash would see.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ {
+++++++++++++ ++ ++++++++ const int stripe_nr = i / rbio->stripe_nsectors;
+++++++++++++ ++ ++++++++ const int sectornr = i % rbio->stripe_nsectors;
+++++++++++++ ++ ++++++++ const u64 logical = rbio->bioc->full_stripe_logical +
+++++++++++++ ++ ++++++++ (u64)stripe_nr * rbio->bioc->stripe_len +
+++++++++++++ ++ ++++++++ (u64)sectornr * fs_info->sectorsize;
+++++++++++++ ++ ++++++++ u64 refs;
+++++++++++++ ++ ++++++++ u64 egen;
+++++++++++++ ++ ++++++++ int ret;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ ret = rmw_block_referenced(fs_info, logical, gen, &refs,
+++++++++++++ ++ ++++++++ &egen);
+++++++++++++ ++ ++++++++ if (ret == -EAGAIN) {
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_unknown);
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (!ret) {
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_ghost);
+++++++++++++ ++ ++++++++ continue;
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_old);
+++++++++++++ ++ ++++++++ btrfs_warn_rl(fs_info,
+++++++++++++ ++ ++++++++"read-modify-write of stripe %llu rewrites parity over committed tree block %llu at generation %llu (refs %llu, extent generation %llu) while writing generation %llu: write hole",
+++++++++++++ ++ ++++++++ rbio->bioc->full_stripe_logical, logical,
+++++++++++++ ++ ++++++++ gen, refs, egen, write_gen);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++/*
+++++++++++++ ++ ++++++++ * Count and audit a metadata or system read-modify-write
+++++++++++++ ++ ++++++++ * (audit_uncovered_rmw()), and warn if it lands in a raid56 group that
+++++++++++++ ++ ++++++++ * stripe_meta does not cover. Silent when stripe_alloc is off.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++static void report_uncovered_rmw(struct btrfs_raid_bio *rbio)
+++++++++++++ ++ ++++++++{
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++++ ++ ++++++++ bool covered = false;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ /* Data groups are btrfs_is_stripe_alloc_bg()'s. */
+++++++++++++ ++ ++++++++ if ((rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA) &&
+++++++++++++ ++ ++++++++ !(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_METADATA))
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw);
+++++++++++++ ++ ++++++++ audit_uncovered_rmw(rbio);
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Every raid56 metadata and system group is a stripe_meta group while
+++++++++++++ ++ ++++++++ * stripe_alloc is on, and stripe_alloc refuses to enable on mixed
+++++++++++++ ++ ++++++++ * groups (btrfs_stripe_alloc_check_support()), so this warning is a
+++++++++++++ ++ ++++++++ * defensive check: it fires only if a tree block group that should be
+++++++++++++ ++ ++++++++ * covered is not.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, rbio->bioc->full_stripe_logical);
+++++++++++++ ++ ++++++++ if (bg) {
+++++++++++++ ++ ++++++++ covered = btrfs_is_stripe_meta_bg(bg);
+++++++++++++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++++ ++ ++++++++ }
+++++++++++++ ++ ++++++++ if (covered)
+++++++++++++ ++ ++++++++ return;
+++++++++++++ ++ ++++++++ btrfs_warn_rl(fs_info,
+++++++++++++ ++ ++++++++"read-modify-write of full stripe %llu: this block group is not covered by stripe_alloc, the raid56 write hole applies to it",
+++++++++++++ ++ ++++++++ rbio->bioc->full_stripe_logical);
+++++++++++++ ++ ++++++++}
+++++++++++++ ++ ++++++++
+++++++++ ++++++++++++++++/*
+++++++++ ++++++++++++++++ * Return true if the rebuild of data sector @sector_nr of @stripe_nr can be
+++++++++ ++++++++++++++++ * trusted enough to write it back. It can when the sector has a csum: the
+++++++++ ++++++++++++++++ * rebuild was verified. Without a csum, it can only when another data sector
+++++++++ ++++++++++++++++ * of the same vertical stripe also failed and has a csum: the two were
+++++++++ ++++++++++++++++ * rebuilt together from P and Q (on RAID6 that is also how a stale sector
+++++++++ ++++++++++++++++ * without a csum is found, see recover_vertical_retry()), and the csum that
+++++++++ ++++++++++++++++ * verified one verifies the other. A lone rebuild without a csum came from
+++++++++ ++++++++++++++++ * parity that nothing checked, and the copy on the disk may still be right.
+++++++++ ++++++++++++++++ */
+++++++++ ++++++++++++++++static bool rmw_repair_verified(struct btrfs_raid_bio *rbio, int stripe_nr,
+++++++++ ++++++++++++++++ int sector_nr)
+++++++++ ++++++++++++++++{
+++++++++ ++++++++++++++++ if (!rbio->csum_bitmap)
+++++++++ ++++++++++++++++ return false;
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ for (int i = 0; i < rbio->nr_data; i++) {
+++++++++ ++++++++++++++++ const int index = rbio_sector_index(rbio, i, sector_nr);
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ if (!test_bit(index, rbio->csum_bitmap))
+++++++++ ++++++++++++++++ continue;
+++++++++ ++++++++++++++++ if (i == stripe_nr || test_bit(index, rbio->error_bitmap))
+++++++++ ++++++++++++++++ return true;
+++++++++ ++++++++++++++++ }
+++++++++ ++++++++++++++++ return false;
+++++++++ ++++++++++++++++}
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++/*
+++++++++ ++++++++++++++++ * After the read phase of a sub-stripe write, return the sectors it repaired
+++++++++ ++++++++++++++++ * that the write phase should also write back, or NULL if there are none.
+++++++++ ++++++++++++++++ *
+++++++++ ++++++++++++++++ * recover_sectors() leaves the error bit set on every sector it rebuilt: a
+++++++++ ++++++++++++++++ * data sector that failed its csum or its read, every sector of a missing
+++++++++ ++++++++++++++++ * device, and the extra stripe that recover_vertical_retry() found stale.
+++++++++ ++++++++++++++++ * The rebuilt sectors are in the stripe pages and the new P and Q are
+++++++++ ++++++++++++++++ * generated from them, so without a write-back the disk copies stay stale
+++++++++ ++++++++++++++++ * until read repair or scrub finds them.
+++++++++ ++++++++++++++++ *
+++++++++ ++++++++++++++++ * Nothing is written to a missing device. A data sector is written back only
+++++++++ ++++++++++++++++ * if its rebuild is verified (rmw_repair_verified()). P and Q are generated
+++++++++ ++++++++++++++++ * from the data either way, so a failed or stale one is always written back.
+++++++++ ++++++++++++++++ *
+++++++++ ++++++++++++++++ * This rbio has not written anything yet, so none of these is a retry of a
+++++++++ ++++++++++++++++ * write that just failed here.
+++++++++ ++++++++++++++++ */
+++++++++ ++++++++++++++++static unsigned long *rmw_repaired_sectors(struct btrfs_raid_bio *rbio)
+++++++++ ++++++++++++++++{
+++++++++ ++++++++++++++++ unsigned long *repaired = NULL;
+++++++++ ++++++++++++++++ int total_sector_nr;
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ for_each_set_bit(total_sector_nr, rbio->error_bitmap, rbio->nr_sectors) {
+++++++++ ++++++++++++++++ const int stripe = total_sector_nr / rbio->stripe_nsectors;
+++++++++ ++++++++++++++++ const int sectornr = total_sector_nr % rbio->stripe_nsectors;
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ if (!rbio->bioc->stripes[stripe].dev->bdev)
+++++++++ ++++++++++++++++ continue;
+++++++++ ++++++++++++++++ if (stripe < rbio->nr_data &&
+++++++++ ++++++++++++++++ !rmw_repair_verified(rbio, stripe, sectornr))
+++++++++ ++++++++++++++++ continue;
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ if (!repaired) {
+++++++++ ++++++++++++++++ /* Without memory the sectors just stay stale on disk. */
+++++++++ ++++++++++++++++ repaired = bitmap_zalloc(rbio->nr_sectors, GFP_NOFS);
+++++++++ ++++++++++++++++ if (!repaired)
+++++++++ ++++++++++++++++ return NULL;
+++++++++ ++++++++++++++++ }
+++++++++ ++++++++++++++++ set_bit(total_sector_nr, repaired);
+++++++++ ++++++++++++++++ }
+++++++++ ++++++++++++++++ return repaired;
+++++++++ ++++++++++++++++}
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++/*
+++++++++ ++++++++++++++++ * Add write bios for the repaired sectors that the write phase does not write
+++++++++ ++++++++++++++++ * anyway: data sectors no bio of this rbio covers, and P/Q outside the
+++++++++ ++++++++++++++++ * vertical stripes being written. The others are dropped from @repaired.
+++++++++ ++++++++++++++++ * The replace target gets the same sectors as its source.
+++++++++ ++++++++++++++++ */
+++++++++ ++++++++++++++++static int rmw_assemble_repair_bios(struct btrfs_raid_bio *rbio,
+++++++++ ++++++++++++++++ struct bio_list *bio_list,
+++++++++ ++++++++++++++++ unsigned long *repaired)
+++++++++ ++++++++++++++++{
+++++++++ ++++++++++++++++ int total_sector_nr;
+++++++++ ++++++++++++++++ int ret;
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ for_each_set_bit(total_sector_nr, repaired, rbio->nr_sectors) {
+++++++++ ++++++++++++++++ const int stripe = total_sector_nr / rbio->stripe_nsectors;
+++++++++ ++++++++++++++++ const int sectornr = total_sector_nr % rbio->stripe_nsectors;
+++++++++ ++++++++++++++++ phys_addr_t *paddrs;
+++++++++ ++++++++++++++++ bool written;
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ if (stripe < rbio->nr_data)
+++++++++ ++++++++++++++++ written = sector_paddrs_in_rbio(rbio, stripe, sectornr, 1);
+++++++++ ++++++++++++++++ else
+++++++++ ++++++++++++++++ written = test_bit(sectornr, &rbio->dbitmap);
+++++++++ ++++++++++++++++ if (written) {
+++++++++ ++++++++++++++++ clear_bit(total_sector_nr, repaired);
+++++++++ ++++++++++++++++ continue;
+++++++++ ++++++++++++++++ }
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ paddrs = rbio_stripe_paddrs(rbio, stripe, sectornr);
+++++++++ ++++++++++++++++ ret = rbio_add_io_paddrs(rbio, bio_list, paddrs, stripe,
+++++++++ ++++++++++++++++ sectornr, REQ_OP_WRITE);
+++++++++ ++++++++++++++++ if (ret)
+++++++++ ++++++++++++++++ goto error;
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ if (likely(!rbio->bioc->replace_nr_stripes) ||
+++++++++ ++++++++++++++++ stripe != rbio->bioc->replace_stripe_src)
+++++++++ ++++++++++++++++ continue;
+++++++++ ++++++++++++++++ ret = rbio_add_io_paddrs(rbio, bio_list, paddrs,
+++++++++ ++++++++++++++++ rbio->real_stripes, sectornr,
+++++++++ ++++++++++++++++ REQ_OP_WRITE);
+++++++++ ++++++++++++++++ if (ret)
+++++++++ ++++++++++++++++ goto error;
+++++++++ ++++++++++++++++ }
+++++++++ ++++++++++++++++ return 0;
+++++++++ ++++++++++++++++error:
+++++++++ ++++++++++++++++ bio_list_put(bio_list);
+++++++++ ++++++++++++++++ return -EIO;
+++++++++ ++++++++++++++++}
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++/*
+++++++++ ++++++++++++++++ * A write-back that fails leaves the sector as stale as it was before this
+++++++++ ++++++++++++++++ * write, so it does not count against the tolerance of the write itself.
+++++++++ ++++++++++++++++ */
+++++++++ ++++++++++++++++static void rmw_finish_repair_bios(struct btrfs_raid_bio *rbio,
+++++++++ ++++++++++++++++ unsigned long *repaired)
+++++++++ ++++++++++++++++{
+++++++++ ++++++++++++++++ int written = 0;
+++++++++ ++++++++++++++++ int failed = 0;
+++++++++ ++++++++++++++++ int total_sector_nr;
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ for_each_set_bit(total_sector_nr, repaired, rbio->nr_sectors) {
+++++++++ ++++++++++++++++ written++;
+++++++++ ++++++++++++++++ if (test_and_clear_bit(total_sector_nr, rbio->error_bitmap))
+++++++++ ++++++++++++++++ failed++;
+++++++++ ++++++++++++++++ }
+++++++++ ++++++++++++++++ if (written)
+++++++++ ++++++++++++++++ btrfs_debug_rl(rbio->bioc->fs_info,
+++++++++ ++++++++++++++++ "sub-stripe write for full stripe %llu wrote back %d repaired sectors, %d failed",
+++++++++ ++++++++++++++++ rbio->bioc->full_stripe_logical, written, failed);
+++++++++ ++++++++++++++++}
+++++++++ ++++++++++++++++
+++++++++ + + ++ ++++++++static void rmw_rbio(struct btrfs_raid_bio *rbio)
+++++++++ + + ++ ++++++++{
+++++++++ + + ++ ++++++++ struct bio_list bio_list;
+++++++++ + + ++ ++++++++ int sectornr;
+++++++++ ++++++++++++++++ unsigned long *repaired = NULL;
+++++++++ + + ++ ++++++++ int ret = 0;
+++++++++ + + ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Write-hole invariant check: every write must land in a stripe
+++++++++++++ ++ ++++++++ * covered by a live stripe run when stripe-exclusive allocation is
+++++++++++++ ++ ++++++++ * enabled. The bios gathered in this rbio have not reported their
+++++++++++++ ++ ++++++++ * IO done yet, so their runs cannot drain under us.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ btrfs_stripe_check_write(rbio->bioc->fs_info,
+++++++++++++ ++ ++++++++ rbio->bioc->full_stripe_logical,
+++++++++++++ ++ ++++++++ !rbio_is_full(rbio));
+++++++++++++ ++ ++++++++
+++++++++ + + ++ ++++++++ /*
+++++++++ + + ++ ++++++++ * Allocate the pages for parity first, as P/Q pages will always be
+++++++++ + + ++ ++++++++ * needed for both full-stripe and sub-stripe writes.
+++++++++ + + ++ ++++++++ */
+++++++++ + + ++ ++++++++ ret = alloc_rbio_parity_pages(rbio);
+++++++++ + + ++ ++++++++ if (ret < 0)
+++++++++ + + ++ ++++++++ goto out;
+++++++++ + + ++ ++++++++
+++++++++ + + ++ ++++++++ /*
+++++++++ + + ++ ++++++++ * Either full stripe write, or we have every data sector already
+++++++++ + + ++ ++++++++ * cached, can go to write path immediately.
+++++++++++++ ++ ++++++++ *
+++++++++++++ ++ ++++++++ * Try padding before trusting the cache: a cached rbio can hold pages
+++++++++++++ ++ ++++++++ * from the stripe's previous life, before its extents were freed and
+++++++++++++ ++ ++++++++ * the stripe re-claimed, and parity computed from them matches the
+++++++++++++ ++ ++++++++ * disk only if every uncovered column still holds that data. Padding
+++++++++++++ ++ ++++++++ * writes every uncovered column, so the stripe needs nothing on disk.
+++++++++ + + ++ ++++++++ */
- - if (!rbio_is_full(rbio) && need_read_stripe_sectors(rbio)) {
+++++++++++++ ++ ++++++++ if (!rbio_is_full(rbio) && !rmw_try_pad_full(rbio) &&
+++++++++++++ ++ ++++++++ need_read_stripe_sectors(rbio)) {
+++++++++++++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++++ ++ ++++++++
+++++++++++++ ++ ++++++++ /*
+++++++++++++ ++ ++++++++ * Every data read-modify-write, parked or not. With
+++++++++++++ ++ ++++++++ * allow_rmw empty and a covered block group this is the
+++++++++++++ ++ ++++++++ * count that should be zero.
+++++++++++++ ++ ++++++++ */
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw);
+++++++++++++ ++ ++++++++ if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA))
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_reads);
+++++++++++++ ++ ++++++++ rbio_count_rmw_reason(rbio);
+++++++++++++ ++ ++++++++ /* A parked-then-flushed write that still had to read. */
+++++++++++++ ++ ++++++++ if (test_bit(RBIO_WAS_PARKED_BIT, &rbio->flags))
+++++++++++++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.rmw_reads);
+++++++++ + + ++ ++++++++ /*
+++++++++ + + ++ ++++++++ * Now we're doing sub-stripe write, also need all data stripes
+++++++++ + + ++ ++++++++ * to do the full RMW.
+++++++++ + + ++ ++++++++ */
+++++++++ + + ++ ++++++++ ret = alloc_rbio_data_pages(rbio);
+++++++++ + + ++ ++++++++ if (ret < 0)
+++++++++ + + ++ ++++++++ goto out;
+++++++++ + + ++ ++++++++
+++++++++ + + ++ ++++++++ index_rbio_pages(rbio);
+++++++++ + + ++ ++++++++
+++++++++ + + ++ ++++++++ ret = rmw_read_wait_recover(rbio);
+++++++++ + + ++ ++++++++ if (ret < 0)
+++++++++ + + ++ ++++++++ goto out;
+++++++++ ++++++++++++++++ repaired = rmw_repaired_sectors(rbio);
+++++++++ + + ++ ++++++++ }
+++++++++ + + ++ ++++++++
+++++++++++++ ++ ++++++++ /* Still a sub-stripe write: this stripe is being modified in place. */
+++++++++++++ ++ ++++++++ if (!rbio_is_full(rbio) && !test_bit(RBIO_PADDED_BIT, &rbio->flags))
+++++++++++++ ++ ++++++++ report_uncovered_rmw(rbio);
+++++++++++++ ++ ++++++++
+++++++++ + + ++ ++++++++ /*
+++++++++ + + ++ ++++++++ * At this stage we're not allowed to add any new bios to the
+++++++++ + + ++ ++++++++ * bio list any more, anyone else that wants to change this stripe
+++++++++ + + ++ ++++++++ * needs to do their own rmw.
+++++++++ + + ++ ++++++++ */
+++++++++ + + ++ ++++++++ spin_lock(&rbio->bio_list_lock);
+++++++++ + + ++ ++++++++ set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
+++++++++ + + ++ ++++++++ spin_unlock(&rbio->bio_list_lock);
+++++++++ + + ++ ++++++++
+++++++++ + + ++ ++++++++ bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors);
index_rbio_pages(rbio);
q_paddr = INVALID_PADDR;
}
- * nothing back to the scrubbed device. A running dev-replace still
- * copies to its target.
+++++++++++ ++++++++++++++ /* A read-only scrub only checks: submit no write. */
+++++++++++ ++++++++++++++ if (rbio->scrub_readonly)
+++++++++++ ++++++++++++++ goto submit_write;
+++++++++++ ++++++++++++++
++++++++++ ++++++++++++++ /*
++++++++++ ++++++++++++++ * After a transaction abort or another filesystem error, write
+++++++++++ ++++++++++++++ * nothing back to the scrubbed devices, rebuilt data included. A
+++++++++++ ++++++++++++++ * running dev-replace still copies parity to its target.
++++++++++ ++++++++++++++ */
++++++++++ ++++++++++++++ if (!btrfs_can_submit_repair_write(rbio->bioc->fs_info))
++++++++++ ++++++++++++++ goto replace;
++++++++++ ++++++++++++++
+++++++++++ ++++++++++++++ /* Write the data sectors rebuilt from P and Q. */
+++++++++++ ++++++++++++++ for_each_set_bit(sectornr, &rebuilt, rbio->stripe_nsectors) {
+++++++++++ ++++++++++++++ const int stripe = rebuilt_stripe[sectornr];
+++++++++++ ++++++++++++++ const struct btrfs_io_stripe *io_stripe = &bioc->stripes[stripe];
+++++++++++ ++++++++++++++
+++++++++++ ++++++++++++++ btrfs_warn_rl(bioc->fs_info,
+++++++++++ ++++++++++++++"scrub: raid6 data at logical %llu on devid %llu physical %llu did not match P and Q, rebuilt it",
+++++++++++ ++++++++++++++ bioc->full_stripe_logical +
+++++++++++ ++++++++++++++ btrfs_stripe_nr_to_offset(stripe, bioc->stripe_len) +
+++++++++++ ++++++++++++++ ((u64)sectornr << bioc->fs_info->sectorsize_bits),
+++++++++++ ++++++++++++++ io_stripe->dev->devid,
+++++++++++ ++++++++++++++ io_stripe->physical +
+++++++++++ ++++++++++++++ ((u64)sectornr << bioc->fs_info->sectorsize_bits));
+++++++++++ ++++++++++++++ ret = rbio_add_io_paddrs(rbio, &bio_list,
+++++++++++ ++++++++++++++ rbio_stripe_paddrs(rbio, stripe, sectornr),
+++++++++++ ++++++++++++++ stripe, sectornr, REQ_OP_WRITE);
+++++++++++ ++++++++++++++ if (ret)
+++++++++++ ++++++++++++++ goto cleanup;
+++++++++++ ++++++++++++++ }
+++++++++++ ++++++++++++++
/*
* time to start writing. Make bios for everything from the
* higher layers (the bio_list in our rbio) and our p/q. Ignore