wait_var_event(&bg->reservations, !atomic_read(&bg->reservations));
}
- - if (bg->start == READ_ONCE(fs_info->data_reloc_bg))
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Open stripe runs
+++++++++++ ++ ++ ++++++++ * ================
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * The raid56 stripe allocation policy only allocates from "open stripe
+++++++++++ ++ ++ ++++++++ * runs": contiguous, stripe-aligned regions claimed whole from the free
+++++++++++ ++ ++ ++++++++ * space cache (btrfs_claim_free_stripe_run()) and filled strictly
+++++++++++ ++ ++ ++++++++ * sequentially. A run is closed when it is exhausted, when it loses a
+++++++++++ ++ ++ ++++++++ * band collision (stripe_run_place()), when a private run is too small and
+++++++++++ ++ ++ ++++++++ * is replaced, when a metadata run belongs to an earlier transaction, or
+++++++++++ ++ ++ ++++++++ * when it is retired at transaction commit. A closed run is never
+++++++++++ ++ ++ ++++++++ * reopened, and its unallocated tail returns to the free space cache
+++++++++++ ++ ++ ++++++++ * where, being part of a partially filled stripe, it cannot be claimed
+++++++++++ ++ ++ ++++++++ * until the whole stripe frees. So every stripe is written within one
+++++++++++ ++ ++ ++++++++ * run's lifetime, which closes the raid56 write hole for these block
+++++++++++ ++ ++ ++++++++ * groups.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Each run counts the bytes reserved from it whose IO has not completed
+++++++++++ ++ ++ ++++++++ * (inflight_bytes). At commit, while the transaction still accepts joins
+++++++++++ ++ ++ ++++++++ * (before TRANS_STATE_COMMIT_DOING), btrfs_retire_open_stripes() closes
+++++++++++ ++ ++ ++++++++ * the data runs opened before the commit and waits for them to drain. The
+++++++++++ ++ ++ ++++++++ * wait must be a pure data-IO wait, never an ordered extent completion
+++++++++++ ++ ++ ++++++++ * wait, and must happen where joins still succeed: a task holding
+++++++++++ ++ ++ ++++++++ * unreported run bytes may block on a join (e.g. a chunk allocation during
+++++++++++ ++ ++ ++++++++ * delalloc). Metadata runs are retired after tree writeback
+++++++++++ ++ ++ ++++++++ * (btrfs_retire_meta_stripes()); persistent NOCOW runs survive commits.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Runs opened while the commit is in progress carry a newer open_seq and
+++++++++++ ++ ++ ++++++++ * belong to the next window. An ordered extent from such a run that
+++++++++++ ++ ++ ++++++++ * completes before the commit ends sees that its run's open_seq is not
+++++++++++ ++ ++ ++++++++ * retired by this transaction and defers its file extent to the next one.
+++++++++++ ++ ++ ++++++++ * The membership add, the sequence read and the run installation happen
+++++++++++ ++ ++ ++++++++ * atomically under fs_info->open_stripe_lock, so every run is either seen
+++++++++++ ++ ++ ++++++++ * by the walk that retires its window or stamped with a sequence that
+++++++++++ ++ ++ ++++++++ * forces the deferral.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Lock order: fs_info->open_stripe_lock outside bg->stripe_run_lock. The
+++++++++++ ++ ++ ++++++++ * stripe_run_lock is taken with interrupts disabled, which no current
+++++++++++ ++ ++ ++++++++ * caller needs: completed IO is reported from bio completion work, which
+++++++++++ ++ ++ ++++++++ * runs in task context. Membership in fs_info->open_stripe_bgs holds a
+++++++++++ ++ ++ ++++++++ * block group reference, dropped by the retire walk once the group has no
+++++++++++ ++ ++ ++++++++ * runs left, or by btrfs_free_open_stripe_runs() at teardown.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++static bool stripe_extents_owned_by(struct btrfs_fs_info *fs_info,
+++++++++++ ++ ++ ++++++++ u64 root_id, u64 ino, u64 start, u64 len,
+++++++++++ ++ ++ ++++++++ u64 *frontier, bool nowait);
+++++++++++ ++ ++ ++++++++static bool stripe_range_run_free(struct btrfs_block_group *bg, u64 start,
+++++++++++ ++ ++ ++++++++ u64 len);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++struct btrfs_open_stripe_run {
+++++++++++ ++ ++ ++++++++ struct list_head list; /* bg->open_stripe_runs */
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ u64 start;
+++++++++++ ++ ++ ++++++++ u64 end; /* exclusive; shrunk to offset on close */
+++++++++++ ++ ++ ++++++++ u64 offset; /* next unallocated byte */
+++++++++++ ++ ++ ++++++++ u64 inflight_bytes; /* reserved bytes with data IO pending */
+++++++++++ ++ ++ ++++++++ u64 open_seq; /* fs_info->stripe_retire_seq at open */
+++++++++++ ++ ++ ++++++++ u64 transid; /* fs_info->generation at open */
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Private runs are found by (root, owner): root is the subvolume's
+++++++++++ ++ ++ ++++++++ * root id, owner the inode number (LOG, NOCOW), a shared LOG key
+++++++++++ ++ ++ ++++++++ * (bit 63 set, see stripe_run_owner()), or for META_LOG the root id
+++++++++++ ++ ++ ++++++++ * again. Inode numbers are only unique within a subvolume.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ u64 root;
+++++++++++ ++ ++ ++++++++ u64 owner;
+++++++++++ ++ ++ ++++++++ enum btrfs_stripe_run_class class;
+++++++++++ ++ ++ ++++++++ bool open; /* accepting allocations */
+++++++++++ ++ ++ ++++++++ bool persistent; /* survives commits; see close_block_group_stripe_runs() */
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Relocation's run: kept after it closes and drains, until relocation
+++++++++++ ++ ++ ++++++++ * releases it (btrfs_release_stripe_reloc_runs()). See
+++++++++++ ++ ++ ++++++++ * run_freed_on_drain().
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ bool retained;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * One bit per sector of the run, set while the sector holds an
+++++++++++ ++ ++ ++++++++ * allocation someone still intends to write. Padding uses it to find
+++++++++++ ++ ++ ++++++++ * dead sectors below the frontier (allocations abandoned before they
+++++++++++ ++ ++ ++++++++ * were written), which are as safe to zero-fill as space above it.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ unsigned long *live;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Sectors of @live whose data has been written (raid56 write
+++++++++++ ++ ++ ++++++++ * completions, btrfs_stripe_run_mark_written()); same size as @live.
+++++++++++ ++ ++ ++++++++ * A stripe with live sectors not yet written is still waiting for
+++++++++++ ++ ++ ++++++++ * writes, and its cached rbio stays pinned (raid56.c, cache_rbio()).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ unsigned long *written;
+++++++++++ ++ ++ ++++++++ u32 live_nbits; /* sectors @live covers; caps growth */
+++++++++++ ++ ++ ++++++++ /* META_LOG: the owning subvolume's log_transid this run belongs to. */
+++++++++++ ++ ++ ++++++++ u64 log_transid;
+++++++++++ ++ ++ ++++++++};
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++static void btrfs_dump_open_stripe_runs(struct btrfs_block_group *bg)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list)
+++++++++++ ++ ++ ++++++++ btrfs_warn(bg->fs_info,
+++++++++++ ++ ++ ++++++++" run %llu..%llu class %d frontier %llu open %d inflight %llu transid %llu",
+++++++++++ ++ ++ ++++++++ run->start, run->end, run->class, run->offset,
+++++++++++ ++ ++ ++++++++ run->open, run->inflight_bytes, run->transid);
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++static bool bg_has_undrained_meta_runs(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ u64 transid)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ bool ret = false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (!btrfs_stripe_run_class_is_meta(run->class))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * An open run here belongs to the running transaction: the
+++++++++++ ++ ++ ++++++++ * retirer closed every run it is responsible for before its
+++++++++++ ++ ++ ++++++++ * wait, so this one was opened while it waited, and drains
+++++++++++ ++ ++ ++++++++ * when that transaction's own commit writes its tree blocks.
+++++++++++ ++ ++ ++++++++ * Waiting on it would be waiting for writes this commit never
+++++++++++ ++ ++ ++++++++ * issued.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (run->open)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Likewise a run the running transaction opened and a band
+++++++++++ ++ ++ ++++++++ * collision or eviction closed early: its blocks are written,
+++++++++++ ++ ++ ++++++++ * and its stripes settled, by that transaction's own commit.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (run->transid > transid)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (run->inflight_bytes) {
+++++++++++ ++ ++ ++++++++ ret = true;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++static bool open_stripe_run_drained(const struct btrfs_open_stripe_run *run)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ return !run->open && run->inflight_bytes == 0;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Round to a full stripe.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * full_stripe_len is nr_data_stripes * stripe_len, which is not a power of
+++++++++++ ++ ++ ++++++++ * two for most raid56 geometries (3 data stripes: 192K), so the mask-based
+++++++++++ ++ ++ ++++++++ * round_down(), round_up() and IS_ALIGNED() must not be used. Divide
+++++++++++ ++ ++ ++++++++ * instead.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static u64 stripe_round_down(u64 val, u64 fsl)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ return div64_u64(val, fsl) * fsl;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++static u64 stripe_round_up(u64 val, u64 fsl)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ return stripe_round_down(val + fsl - 1, fsl);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++static bool stripe_is_aligned(u64 val, u64 fsl)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ return stripe_round_down(val, fsl) == val;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Band for a run with @remaining bytes of unallocated space: the power-of-two
+++++++++++ ++ ++ ++++++++ * bucket of its remaining sectors (band k == remaining in [2^k, 2^(k+1))
+++++++++++ ++ ++ ++++++++ * sectors). Runs are kept at most one per band so an allocation can take the
+++++++++++ ++ ++ ++++++++ * smallest band that fits.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static int stripe_run_band(const struct btrfs_block_group *bg, u64 remaining)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ u32 sectors = remaining >> bg->fs_info->sectorsize_bits;
+++++++++++ ++ ++ ++++++++ int band;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!sectors)
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++ band = fls(sectors) - 1;
+++++++++++ ++ ++ ++++++++ if (band >= BTRFS_STRIPE_RUN_NR_BANDS)
+++++++++++ ++ ++ ++++++++ band = BTRFS_STRIPE_RUN_NR_BANDS - 1;
+++++++++++ ++ ++ ++++++++ return band;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Growth headroom for a run's liveness map, in full stripes.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * The map is sized once and never reallocated, because the only place a run
+++++++++++ ++ ++ ++++++++ * grows -- the growable frontier in btrfs_alloc_from_open_stripe() -- holds
+++++++++++ ++ ++ ++++++++ * bg->stripe_run_lock with interrupts disabled, where nothing may sleep.
+++++++++++ ++ ++ ++++++++ * Sizing it beyond the run's initial span is what lets that path work at
+++++++++++ ++ ++ ++++++++ * all; a run that would outgrow its map stops growing instead, and the
+++++++++++ ++ ++ ++++++++ * allocation opens a fresh run.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++#define BTRFS_STRIPE_RUN_LIVE_STRIPES 32
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Allocate a run's liveness map. Must be called before taking
+++++++++++ ++ ++ ++++++++ * bg->stripe_run_lock: this sleeps. Sets *nbits to what the map covers.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static unsigned long *alloc_run_live_map(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ u64 run_len, u32 *nbits)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ const u64 span = max_t(u64, run_len,
+++++++++++ ++ ++ ++++++++ (u64)BTRFS_STRIPE_RUN_LIVE_STRIPES *
+++++++++++ ++ ++ ++++++++ bg->full_stripe_len);
+++++++++++ ++ ++ ++++++++ unsigned long *map;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ *nbits = span >> bg->fs_info->sectorsize_bits;
+++++++++++ ++ ++ ++++++++ map = bitmap_zalloc(*nbits, GFP_NOFS);
+++++++++++ ++ ++ ++++++++ if (!map)
+++++++++++ ++ ++ ++++++++ *nbits = 0;
+++++++++++ ++ ++ ++++++++ return map;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/* Mark [start, start+len) live (allocated) or dead (abandoned). */
+++++++++++ ++ ++ ++++++++static void run_live_set(struct btrfs_open_stripe_run *run, u64 start, u64 len,
+++++++++++ ++ ++ ++++++++ bool live)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
+++++++++++ ++ ++ ++++++++ u32 first, nr;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!run->live || !run->live_nbits || start < run->start)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ first = (start - run->start) >> bits;
+++++++++++ ++ ++ ++++++++ nr = len >> bits;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Growth is capped at the map size, so this should be unreachable. If
+++++++++++ ++ ++ ++++++++ * it happens, retire the map: btrfs_stripe_run_pad_mask() then declines
+++++++++++ ++ ++ ++++++++ * the run, and the write falls back to read-modify-write instead of
+++++++++++ ++ ++ ++++++++ * zero-filling a sector that may be live.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (unlikely(first + nr > run->live_nbits)) {
+++++++++++ ++ ++ ++++++++ WARN_ON_ONCE(1);
+++++++++++ ++ ++ ++++++++ run->live_nbits = 0;
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (live)
+++++++++++ ++ ++ ++++++++ bitmap_set(run->live, first, nr);
+++++++++++ ++ ++ ++++++++ else
+++++++++++ ++ ++ ++++++++ bitmap_clear(run->live, first, nr);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * A run is dead when every byte it handed out has been freed again
+++++++++++ ++ ++ ++++++++ * (btrfs_stripe_run_note_free() clears the live bits). It then protects no
+++++++++++ ++ ++ ++++++++ * data, and the whole free stripes inside its range stay unclaimable
+++++++++++ ++ ++ ++++++++ * (btrfs_stripe_run_range_usable()) until it is closed.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Requires inflight_bytes == 0, since an in-flight write must keep its
+++++++++++ ++ ++ ++++++++ * stripes, and a valid live map (live_nbits == 0 means liveness is
+++++++++++ ++ ++ ++++++++ * unknown). Caller holds bg->stripe_run_lock.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static bool open_stripe_run_dead(const struct btrfs_open_stripe_run *run)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ if (run->inflight_bytes)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ if (run->offset <= run->start)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ if (!run->live || !run->live_nbits)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ return bitmap_empty(run->live, run->live_nbits);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Whether a run is freed as soon as it drains. Two kinds must outlive the
+++++++++++ ++ ++ ++++++++ * drain, because later writes to their stripes still need a covering run to
+++++++++++ ++ ++ ++++++++ * park against or pad from:
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * - Metadata runs: writeback may submit a stripe's tree blocks as separate
+++++++++++ ++ ++ ++++++++ * rbios, so the run drains after the first one. They are freed by
+++++++++++ ++ ++ ++++++++ * btrfs_retire_meta_stripes() once every tree block is on disk.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * - Relocation runs: relocation preallocates its destination extents (no
+++++++++++ ++ ++ ++++++++ * IO, so the run drains at once) and writes them only after the commit
+++++++++++ ++ ++ ++++++++ * that ends relocate_block_group(). They are retained until relocation
+++++++++++ ++ ++ ++++++++ * releases them (btrfs_release_stripe_reloc_runs()). Meanwhile their
+++++++++++ ++ ++ ++++++++ * stripes hold only relocation's own copies, and the closed run keeps its
+++++++++++ ++ ++ ++++++++ * range unclaimable.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static bool run_freed_on_drain(const struct btrfs_open_stripe_run *run)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ return !btrfs_stripe_run_class_is_meta(run->class) && !run->retained;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++static void free_open_stripe_run(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ ASSERT(open_stripe_run_drained(run));
+++++++++++ ++ ++ ++++++++ list_del(&run->list);
+++++++++++ ++ ++ ++++++++ bitmap_free(run->live);
+++++++++++ ++ ++ ++++++++ bitmap_free(run->written);
+++++++++++ ++ ++ ++++++++ kfree(run);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Wholly free stripes inside this run's range were counted as trapped
+++++++++++ ++ ++ ++++++++ * by the last stripe_unusable scan (btrfs_stripe_run_stranded_stripes());
+++++++++++ ++ ++ ++++++++ * with the run gone they are claimable again, so have the next commit
+++++++++++ ++ ++ ++++++++ * rescan the group and credit them back.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (READ_ONCE(bg->stripe_unusable_ready)) {
+++++++++++ ++ ++ ++++++++ set_bit(BLOCK_GROUP_FLAG_STRIPE_UNUSABLE_DIRTY, &bg->runtime_flags);
+++++++++++ ++ ++ ++++++++ atomic64_inc(&bg->fs_info->stripe_run_stats.strand_rescan_marks);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * A run has drained. Free it if nothing needs it any more; otherwise
+++++++++++ ++ ++ ++++++++ * (metadata or retained relocation runs) wake the waiters directly, since a
+++++++++++ ++ ++ ++++++++ * report can arrive after btrfs_retire_meta_stripes() started waiting, e.g.
+++++++++++ ++ ++ ++++++++ * when an unwritten tree block is freed.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void open_stripe_run_drain_reported(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ if (!open_stripe_run_drained(run))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ if (run_freed_on_drain(run))
+++++++++++ ++ ++ ++++++++ free_open_stripe_run(bg, run);
+++++++++++ ++ ++ ++++++++ else
+++++++++++ ++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Close a run: no further allocation may join it, and its unallocated tail
+++++++++++ ++ ++ ++++++++ * must be returned to the free space cache by the caller (after dropping the
+++++++++++ ++ ++ ++++++++ * lock). The caller has already removed @run from bg->open_stripe[][].
+++++++++++ ++ ++ ++++++++ * Caller holds bg->stripe_run_lock. Returns the tail length, sets *tail_start.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static u64 close_open_stripe_run(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run,
+++++++++++ ++ ++ ++++++++ u64 *tail_start)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ u64 tail_len;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ ASSERT(run->open);
+++++++++++ ++ ++ ++++++++ run->open = false;
+++++++++++ ++ ++ ++++++++ *tail_start = run->offset;
+++++++++++ ++ ++ ++++++++ tail_len = run->end - run->offset;
+++++++++++ ++ ++ ++++++++ run->end = run->offset;
+++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= tail_len;
+++++++++++ ++ ++ ++++++++ if (open_stripe_run_drained(run) && !run->retained)
+++++++++++ ++ ++ ++++++++ free_open_stripe_run(bg, run);
+++++++++++ ++ ++ ++++++++ return tail_len;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Fold this block group's open-run remainder delta into the space_info
+++++++++++ ++ ++ ++++++++ * counter that reservations see. Bytes inside an open run are invisible
+++++++++++ ++ ++ ++++++++ * to the free space cache (the claim removed them) and will either be
+++++++++++ ++ ++ ++++++++ * allocated or become trapped tails when the run closes, so the data
+++++++++++ ++ ++ ++++++++ * reservation path must not admit writes against them: a reservation
+++++++++++ ++ ++ ++++++++ * that cannot allocate at writeback time discards the pages silently.
+++++++++++ ++ ++ ++++++++ * Called after every remainder mutation. Compute and apply are one
+++++++++++ ++ ++ ++++++++ * critical section (sinfo->lock nests inside stripe_run_lock): applied
+++++++++++ ++ ++ ++++++++ * out of order, a negative delta could overtake the positive it was
+++++++++++ ++ ++ ++++++++ * computed against and transiently underflow the space_info sum.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void stripe_open_remainder_sync(struct btrfs_block_group *bg)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_space_info *sinfo = bg->space_info;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ s64 delta;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ delta = (s64)(bg->stripe_open_remainder - bg->stripe_open_reported);
+++++++++++ ++ ++ ++++++++ bg->stripe_open_reported = bg->stripe_open_remainder;
+++++++++++ ++ ++ ++++++++ if (delta && sinfo) {
+++++++++++ ++ ++ ++++++++ spin_lock(&sinfo->lock);
+++++++++++ ++ ++ ++++++++ if (delta < 0 && sinfo->bytes_stripe_open < (u64)-delta) {
+++++++++++ ++ ++ ++++++++ WARN_ON_ONCE(1);
+++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_open = 0;
+++++++++++ ++ ++ ++++++++ } else {
+++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_open += delta;
+++++++++++ ++ ++ ++++++++ }
++++++++++++++ +++++++++++ /*
++++++++++++++ +++++++++++ * A claimed run's remainder left the free space cache, but
++++++++++++++ +++++++++++ * reloc_reserve_free() still counts it: move it out of a held
++++++++++++++ +++++++++++ * group's share with it, or bytes_stripe_open and
++++++++++++++ +++++++++++ * bytes_reloc_reserve both count it until the next commit's
++++++++++++++ +++++++++++ * resync. Every writer of the share holds sinfo->lock.
++++++++++++++ +++++++++++ */
++++++++++++++ +++++++++++ btrfs_reloc_reserve_adjust(bg, -delta);
+++++++++++ ++ ++ ++++++++ spin_unlock(&sinfo->lock);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Install @run (not currently in any band slot) into its remaining-space band.
+++++++++++ ++ ++ ++++++++ * If a run already occupies that band, keep whichever has more remaining -- it
+++++++++++ ++ ++ ++++++++ * can serve larger future allocations, and closing the other traps less -- and
+++++++++++ ++ ++ ++++++++ * close the loser, returning its tail via *tail_start. Caller holds
+++++++++++ ++ ++ ++++++++ * bg->stripe_run_lock. Returns the closed tail length, or 0 if no collision.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static u64 stripe_run_place(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run, u64 *tail_start)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ int class = run->class;
+++++++++++ ++ ++ ++++++++ int band = stripe_run_band(bg, run->end - run->offset);
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *occ = bg->open_stripe[class][band];
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Private runs (LOG, NOCOW, META_LOG) never occupy a band slot. */
+++++++++++ ++ ++ ++++++++ ASSERT(class != BTRFS_STRIPE_RUN_LOG);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (occ) {
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *keep, *shut;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (run->end - run->offset >= occ->end - occ->offset) {
+++++++++++ ++ ++ ++++++++ keep = run;
+++++++++++ ++ ++ ++++++++ shut = occ;
+++++++++++ ++ ++ ++++++++ } else {
+++++++++++ ++ ++ ++++++++ keep = occ;
+++++++++++ ++ ++ ++++++++ shut = run;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = keep;
+++++++++++ ++ ++ ++++++++ return close_open_stripe_run(bg, shut, tail_start);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = run;
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Allocate num_bytes from the block group's shared open runs of @class,
+++++++++++ ++ ++ ++++++++ * best fit across the remaining-space bands, claiming a new run via
+++++++++++ ++ ++ ++++++++ * btrfs_claim_free_stripe_run() when none fits. If the freshly claimed
+++++++++++ ++ ++ ++++++++ * stripes directly follow an open run of the same class, that run grows to
+++++++++++ ++ ++ ++++++++ * absorb them instead (see the comment at the claim site). The allocation
+++++++++++ ++ ++ ++++++++ * counts toward the run's inflight bytes until its IO is reported done or
+++++++++++ ++ ++ ++++++++ * abandoned.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in
+++++++++++ ++ ++ ++++++++ * this block group can satisfy the allocation, or -ENOMEM.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++int btrfs_alloc_from_open_stripe(struct btrfs_block_group *bg, u64 num_bytes,
+++++++++++ ++ ++ ++++++++ enum btrfs_stripe_run_class class,
+++++++++++ ++ ++ ++++++++ u64 *ret_offset, u64 *available)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++ ++ ++ ++++++++ const u64 transid = btrfs_get_fs_generation(fs_info);
+++++++++++ ++ ++ ++++++++ u64 evict_start[BTRFS_STRIPE_RUN_NR_BANDS];
+++++++++++ ++ ++ ++++++++ u64 evict_len[BTRFS_STRIPE_RUN_NR_BANDS];
+++++++++++ ++ ++ ++++++++ int nr_evict = 0;
+++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *new_run;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ u64 tail_start = 0;
+++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
+++++++++++ ++ ++ ++++++++ u64 start;
+++++++++++ ++ ++ ++++++++ u64 len;
+++++++++++ ++ ++ ++++++++ int band;
+++++++++++ ++ ++ ++++++++ int ret;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ ASSERT(num_bytes);
+++++++++++ ++ ++ ++++++++ *available = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Best fit: take the run in the smallest band that can hold this
+++++++++++ ++ ++ ++++++++ * allocation, and track the largest remaining across all bands as the
+++++++++++ ++ ++ ++++++++ * retry hint should nothing fit.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ run = NULL;
+++++++++++ ++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) {
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *r = bg->open_stripe[class][band];
+++++++++++ ++ ++ ++++++++ u64 rem;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!r)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Metadata runs are closed after the next transaction is
+++++++++++ ++ ++ ++++++++ * already running (btrfs_retire_meta_stripes()), so a slot can
+++++++++++ ++ ++ ++++++++ * still hold the previous transaction's run. Joining it would
+++++++++++ ++ ++ ++++++++ * put two transactions' tree blocks in the same stripes, and
+++++++++++ ++ ++ ++++++++ * the later one's writeback would read-modify-write parity
+++++++++++ ++ ++ ++++++++ * over the earlier one's committed blocks: the write hole.
+++++++++++ ++ ++ ++++++++ * Close it; its tail returns to the free space cache below.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_META && r->transid != transid) {
+++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++ ++ ++ ++++++++ evict_len[nr_evict] =
+++++++++++ ++ ++ ++++++++ close_open_stripe_run(bg, r,
+++++++++++ ++ ++ ++++++++ &evict_start[nr_evict]);
+++++++++++ ++ ++ ++++++++ if (evict_len[nr_evict])
+++++++++++ ++ ++ ++++++++ nr_evict++;
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ rem = r->end - r->offset;
+++++++++++ ++ ++ ++++++++ if (rem > *available)
+++++++++++ ++ ++ ++++++++ *available = rem;
+++++++++++ ++ ++ ++++++++ if (!run && num_bytes <= rem) {
+++++++++++ ++ ++ ++++++++ run = r;
+++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (run) {
+++++++++++ ++ ++ ++++++++ *ret_offset = run->offset;
+++++++++++ ++ ++ ++++++++ run_live_set(run, run->offset, num_bytes, true);
+++++++++++ ++ ++ ++++++++ run->offset += num_bytes;
+++++++++++ ++ ++ ++++++++ run->inflight_bytes += num_bytes;
+++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
+++++++++++ ++ ++ ++++++++ if (run->offset == run->end)
+++++++++++ ++ ++ ++++++++ run->open = false; /* full: closed, no tail */
+++++++++++ ++ ++ ++++++++ else
+++++++++++ ++ ++ ++++++++ tail_len = stripe_run_place(bg, run, &tail_start);
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (tail_len)
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++ ++ ++ ++++++++ for (int i = 0; i < nr_evict; i++)
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, evict_start[i], evict_len[i]);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ for (int i = 0; i < nr_evict; i++)
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, evict_start[i], evict_len[i]);
+++++++++++ ++ ++ ++++++++ if (nr_evict)
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Zeroed, so the error paths can free ->live and ->written. */
+++++++++++ ++ ++ ++++++++ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
+++++++++++ ++ ++ ++++++++ if (!new_run)
+++++++++++ ++ ++ ++++++++ return -ENOMEM;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ ret = btrfs_claim_free_stripe_run(bg,
+++++++++++ ++ ++ ++++++++ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
+++++++++++ ++ ++ ++++++++ &start, &len);
+++++++++++ ++ ++ ++++++++ if (ret)
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ if (len < num_bytes) {
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Only a shorter run is free. Put it back untouched (it is
+++++++++++ ++ ++ ++++++++ * still fully-free stripes) and let the caller retry with a
+++++++++++ ++ ++ ++++++++ * smaller allocation.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++ ++ ++ ++++++++ *available = max(*available, len);
+++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Sleeps, so it has to happen before the locks below. */
+++++++++++ ++ ++ ++++++++ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
+++++++++++ ++ ++ ++++++++ new_run->written = new_run->live_nbits ?
+++++++++++ ++ ++ ++++++++ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Install under fs_info->open_stripe_lock so that the block group
+++++++++++ ++ ++ ++++++++ * membership, the retire sequence read and the run installation are
+++++++++++ ++ ++ ++++++++ * one atomic step against the commit's retire walk (which bumps the
+++++++++++ ++ ++ ++++++++ * sequence and snapshots the membership list under the same lock).
+++++++++++ ++ ++ ++++++++ * A run is therefore either visible to the walk that retires its
+++++++++++ ++ ++ ++++++++ * window, or stamped with a sequence that walk's transaction did
+++++++++++ ++ ++ ++++++++ * not retire, which defers its extents to the next transaction.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ if (list_empty(&bg->open_stripe_bg_list)) {
+++++++++++ ++ ++ ++++++++ btrfs_get_block_group(bg);
+++++++++++ ++ ++ ++++++++ list_add_tail(&bg->open_stripe_bg_list,
+++++++++++ ++ ++ ++++++++ &fs_info->open_stripe_bgs);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (bg->ro) {
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Lost a race with this block group turning read-only; run
+++++++++++ ++ ++ ++++++++ * retirement for it may already have happened, so a new run
+++++++++++ ++ ++ ++++++++ * must not be installed.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++ ++ ++ ++++++++ *available = 0;
+++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Growable frontier: if the claimed region directly follows an open run
+++++++++++ ++ ++ ++++++++ * of the same class, grow that run over it and place this allocation at
+++++++++++ ++ ++ ++++++++ * the run's old tail, so the tail is used rather than stranded when the
+++++++++++ ++ ++ ++++++++ * run closes. Stripe exclusivity holds: the grown stripes were claimed
+++++++++++ ++ ++ ++++++++ * fully free, and the run is still retired at commit. The run keeps
+++++++++++ ++ ++ ++++++++ * its open_seq: it can only still be in a band slot if the retire walk
+++++++++++ ++ ++ ++++++++ * for its window has not run, which this lock nesting excludes.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) {
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *r = bg->open_stripe[class][band];
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!r || r->end != start)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /* Never grow another transaction's run (see the scan above). */
+++++++++++ ++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_META && r->transid != transid)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Only grow as far as the run's liveness map reaches; it
+++++++++++ ++ ++ ++++++++ * cannot be reallocated here (see alloc_run_live_map()).
+++++++++++ ++ ++ ++++++++ * Beyond that, leave the run alone and open a new one.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (((start + len - r->start) >> fs_info->sectorsize_bits) >
+++++++++++ ++ ++ ++++++++ r->live_nbits)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ ASSERT(r->open);
+++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++ ++ ++ ++++++++ *ret_offset = r->offset;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Mark the handed-out bytes live before anything can consult
+++++++++++ ++ ++ ++++++++ * the map: this allocation begins at the old tail, usually
+++++++++++ ++ ++ ++++++++ * mid-stripe, and a sector the map calls dead is a sector a
+++++++++++ ++ ++ ++++++++ * neighbour's padded write may zero-fill.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ run_live_set(r, r->offset, num_bytes, true);
+++++++++++ ++ ++ ++++++++ r->end = start + len;
+++++++++++ ++ ++ ++++++++ r->offset += num_bytes;
+++++++++++ ++ ++ ++++++++ r->inflight_bytes += num_bytes;
+++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder += len;
+++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
+++++++++++ ++ ++ ++++++++ if (r->offset == r->end)
+++++++++++ ++ ++ ++++++++ r->open = false; /* full: closed, no tail */
+++++++++++ ++ ++ ++++++++ else
+++++++++++ ++ ++ ++++++++ tail_len = stripe_run_place(bg, r, &tail_start);
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ if (tail_len)
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ ret = 0;
+++++++++++ ++ ++ ++++++++ goto out; /* frees the unused new_run */
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ new_run->bg = bg;
+++++++++++ ++ ++ ++++++++ new_run->class = class;
+++++++++++ ++ ++ ++++++++ new_run->persistent = false;
+++++++++++ ++ ++ ++++++++ new_run->retained = class == BTRFS_STRIPE_RUN_RELOC;
+++++++++++ ++ ++ ++++++++ if (new_run->retained)
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.reloc_run_retained);
+++++++++++ ++ ++ ++++++++ new_run->owner = 0;
+++++++++++ ++ ++ ++++++++ new_run->transid = transid;
+++++++++++ ++ ++ ++++++++ new_run->start = start;
+++++++++++ ++ ++ ++++++++ new_run->end = start + len;
+++++++++++ ++ ++ ++++++++ new_run->offset = start + num_bytes;
+++++++++++ ++ ++ ++++++++ new_run->inflight_bytes = num_bytes;
+++++++++++ ++ ++ ++++++++ new_run->open_seq = fs_info->stripe_retire_seq;
+++++++++++ ++ ++ ++++++++ new_run->open = (new_run->offset != new_run->end);
+++++++++++ ++ ++ ++++++++ run_live_set(new_run, start, num_bytes, true);
+++++++++++ ++ ++ ++++++++ list_add_tail(&new_run->list, &bg->open_stripe_runs);
+++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder += new_run->end - new_run->offset;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Place the new run in its remaining-space band, closing whichever run
+++++++++++ ++ ++ ++++++++ * loses a band collision (the last, fuller opener wins). Other bands'
+++++++++++ ++ ++ ++++++++ * open runs are untouched.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (new_run->open)
+++++++++++ ++ ++ ++++++++ tail_len = stripe_run_place(bg, new_run, &tail_start);
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (tail_len)
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ *ret_offset = start;
+++++++++++ ++ ++ ++++++++ new_run = NULL;
+++++++++++ ++ ++ ++++++++ ret = 0;
+++++++++++ ++ ++ ++++++++out:
+++++++++++ ++ ++ ++++++++ /* Unused on every path that did not install it, including errors. */
+++++++++++ ++ ++ ++++++++ if (new_run) {
+++++++++++ ++ ++ ++++++++ bitmap_free(new_run->live);
+++++++++++ ++ ++ ++++++++ bitmap_free(new_run->written);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ kfree(new_run);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Allocate a log tree block from the private run of this log commit.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * The run is keyed by @owner (the subvolume's root id) and @log_transid, so
+++++++++++ ++ ++ ++++++++ * each log commit gets whole stripes of its own: its blocks are durable once
+++++++++++ ++ ++ ++++++++ * fsync returns, and no later writer may share a stripe with them (see
+++++++++++ ++ ++ ++++++++ * BTRFS_STRIPE_RUN_META_LOG). An open run of the same owner from an earlier
+++++++++++ ++ ++ ++++++++ * log commit is closed, as a stale-transid metadata run is in
+++++++++++ ++ ++ ++++++++ * btrfs_alloc_from_open_stripe(). Like the other private runs, it is never
+++++++++++ ++ ++ ++++++++ * placed in a band slot.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in this
+++++++++++ ++ ++ ++++++++ * block group can satisfy the allocation, or -ENOMEM.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++int btrfs_alloc_from_log_stripe_run(struct btrfs_block_group *bg, u64 num_bytes,
+++++++++++ ++ ++ ++++++++ u64 owner, u64 log_transid,
+++++++++++ ++ ++ ++++++++ u64 *ret_offset, u64 *available)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++ ++ ++ ++++++++ const enum btrfs_stripe_run_class class = BTRFS_STRIPE_RUN_META_LOG;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *new_run;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ u64 tail_start = 0;
+++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
+++++++++++ ++ ++ ++++++++ u64 start;
+++++++++++ ++ ++ ++++++++ u64 len;
+++++++++++ ++ ++ ++++++++ int ret;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ ASSERT(num_bytes);
+++++++++++ ++ ++ ++++++++ *available = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (run->class != class || run->owner != owner || !run->open)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * A previous log commit of the same subvolume. Its blocks are
+++++++++++ ++ ++ ++++++++ * durable, so this one must not continue into them; close it
+++++++++++ ++ ++ ++++++++ * and claim fresh stripes below. log_transid starts over with
+++++++++++ ++ ++ ++++++++ * every transaction's log tree, so a run from an earlier
+++++++++++ ++ ++ ++++++++ * transaction can carry the same log_transid: the transid
+++++++++++ ++ ++ ++++++++ * tells them apart. Joining it would hand that transaction's
+++++++++++ ++ ++ ++++++++ * commit, which retires the run and waits for its blocks, a
+++++++++++ ++ ++ ++++++++ * block only this transaction will write.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (run->log_transid != log_transid ||
+++++++++++ ++ ++ ++++++++ run->transid != btrfs_get_fs_generation(fs_info)) {
+++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (num_bytes <= run->end - run->offset) {
+++++++++++ ++ ++ ++++++++ *ret_offset = run->offset;
+++++++++++ ++ ++ ++++++++ run_live_set(run, run->offset, num_bytes, true);
+++++++++++ ++ ++ ++++++++ run->offset += num_bytes;
+++++++++++ ++ ++ ++++++++ run->inflight_bytes += num_bytes;
+++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
+++++++++++ ++ ++ ++++++++ if (run->offset == run->end)
+++++++++++ ++ ++ ++++++++ run->open = false; /* full: closed, no tail */
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ /* Too small for this block: replace it. */
+++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (tail_len) {
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Zeroed: the error paths below free ->live and must find NULL. */
+++++++++++ ++ ++ ++++++++ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
+++++++++++ ++ ++ ++++++++ if (!new_run)
+++++++++++ ++ ++ ++++++++ return -ENOMEM;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ ret = btrfs_claim_free_stripe_run(bg,
+++++++++++ ++ ++ ++++++++ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
+++++++++++ ++ ++ ++++++++ &start, &len);
+++++++++++ ++ ++ ++++++++ if (ret)
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ if (len < num_bytes) {
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++ ++ ++ ++++++++ *available = max(*available, len);
+++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Sleeps, so it has to happen before the locks below. */
+++++++++++ ++ ++ ++++++++ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
+++++++++++ ++ ++ ++++++++ new_run->written = new_run->live_nbits ?
+++++++++++ ++ ++ ++++++++ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ if (list_empty(&bg->open_stripe_bg_list)) {
+++++++++++ ++ ++ ++++++++ btrfs_get_block_group(bg);
+++++++++++ ++ ++ ++++++++ list_add_tail(&bg->open_stripe_bg_list,
+++++++++++ ++ ++ ++++++++ &fs_info->open_stripe_bgs);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (bg->ro) {
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++ ++ ++ ++++++++ *available = 0;
+++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ new_run->bg = bg;
+++++++++++ ++ ++ ++++++++ new_run->class = class;
+++++++++++ ++ ++ ++++++++ new_run->persistent = false;
+++++++++++ ++ ++ ++++++++ new_run->root = owner;
+++++++++++ ++ ++ ++++++++ new_run->owner = owner;
+++++++++++ ++ ++ ++++++++ new_run->log_transid = log_transid;
+++++++++++ ++ ++ ++++++++ new_run->transid = btrfs_get_fs_generation(fs_info);
+++++++++++ ++ ++ ++++++++ new_run->start = start;
+++++++++++ ++ ++ ++++++++ new_run->end = start + len;
+++++++++++ ++ ++ ++++++++ new_run->offset = start + num_bytes;
+++++++++++ ++ ++ ++++++++ new_run->inflight_bytes = num_bytes;
+++++++++++ ++ ++ ++++++++ new_run->open_seq = fs_info->stripe_retire_seq;
+++++++++++ ++ ++ ++++++++ new_run->open = (new_run->offset != new_run->end);
+++++++++++ ++ ++ ++++++++ run_live_set(new_run, start, num_bytes, true);
+++++++++++ ++ ++ ++++++++ list_add_tail(&new_run->list, &bg->open_stripe_runs);
+++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder += new_run->end - new_run->offset;
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.log_run_opened);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ *ret_offset = start;
+++++++++++ ++ ++ ++++++++ new_run = NULL;
+++++++++++ ++ ++ ++++++++ ret = 0;
+++++++++++ ++ ++ ++++++++out:
+++++++++++ ++ ++ ++++++++ if (new_run) {
+++++++++++ ++ ++ ++++++++ bitmap_free(new_run->live);
+++++++++++ ++ ++ ++++++++ bitmap_free(new_run->written);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ kfree(new_run);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * The owner a private run is found by: the inode, or with
+++++++++++ ++ ++ ++++++++ * stripe_log_data_shared a LOG-class run shared by every inode of the
+++++++++++ ++ ++ ++++++++ * subvolume logging in the same log transaction. An allocation of a full
+++++++++++ ++ ++ ++++++++ * stripe or more keeps the inode's private run, so small fsync extents never
+++++++++++ ++ ++ ++++++++ * share the partial last stripe of a large one and wait for its IO. Bit 63
+++++++++++ ++ ++ ++++++++ * keeps the shared keys apart from inode numbers; the subvolume is matched
+++++++++++ ++ ++ ++++++++ * separately, by run->root.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static u64 stripe_run_owner(struct btrfs_inode *inode,
+++++++++++ ++ ++ ++++++++ enum btrfs_stripe_run_class class, u64 num_bytes,
+++++++++++ ++ ++ ++++++++ u64 fsl)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_root *root = inode->root;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_LOG && num_bytes < fsl &&
+++++++++++ ++ ++ ++++++++ READ_ONCE(root->fs_info->stripe_log_data_shared))
+++++++++++ ++ ++ ++++++++ return BIT_ULL(63) | (u32)READ_ONCE(root->log_transid);
+++++++++++ ++ ++ ++++++++ return btrfs_ino(inode);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Allocate num_bytes for @inode from its private run of @class (LOG or
+++++++++++ ++ ++ ++++++++ * NOCOW) in this block group, claiming a fresh run when there is none or it
+++++++++++ ++ ++ ++++++++ * does not fit. With stripe_log_data_shared, small LOG allocations use the
+++++++++++ ++ ++ ++++++++ * run the subvolume shares for the current log transaction
+++++++++++ ++ ++ ++++++++ * (stripe_run_owner()). Private runs are never placed in a band slot and
+++++++++++ ++ ++ ++++++++ * are found by subvolume and owner, so a log commit settles only stripes
+++++++++++ ++ ++ ++++++++ * holding its own extents. A NOCOW allocation may instead re-adopt the
+++++++++++ ++ ++ ++++++++ * free tail of the inode's last partial stripe (@readopt_hint).
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * A run too small for this allocation is closed at once rather than left
+++++++++++ ++ ++ ++++++++ * to strand its tail until commit. Two racing allocations for one owner
+++++++++++ ++ ++ ++++++++ * may each claim a run; the spare drains at the next commit, costing a
+++++++++++ ++ ++ ++++++++ * trapped tail, not correctness.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in
+++++++++++ ++ ++ ++++++++ * this block group can satisfy the allocation, or -ENOMEM.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++int btrfs_alloc_from_inode_stripe_run(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ struct btrfs_inode *inode,
+++++++++++ ++ ++ ++++++++ enum btrfs_stripe_run_class class,
+++++++++++ ++ ++ ++++++++ u64 num_bytes, u64 readopt_hint,
+++++++++++ ++ ++ ++++++++ u64 *ret_offset, u64 *available)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++ ++ ++ ++++++++ const u64 root_id = btrfs_root_id(inode->root);
+++++++++++ ++ ++ ++++++++ const u64 ino = stripe_run_owner(inode, class, num_bytes, fsl);
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *new_run;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ u64 tail_start = 0;
+++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
+++++++++++ ++ ++ ++++++++ u64 start;
+++++++++++ ++ ++ ++++++++ u64 len;
+++++++++++ ++ ++ ++++++++ int ret;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ ASSERT(num_bytes);
+++++++++++ ++ ++ ++++++++ *available = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (run->class != class || run->root != root_id ||
+++++++++++ ++ ++ ++++++++ run->owner != ino || !run->open)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (num_bytes <= run->end - run->offset) {
+++++++++++ ++ ++ ++++++++ *ret_offset = run->offset;
+++++++++++ ++ ++ ++++++++ run_live_set(run, run->offset, num_bytes, true);
+++++++++++ ++ ++ ++++++++ run->offset += num_bytes;
+++++++++++ ++ ++ ++++++++ run->inflight_bytes += num_bytes;
+++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= num_bytes;
+++++++++++ ++ ++ ++++++++ if (run->offset == run->end)
+++++++++++ ++ ++ ++++++++ run->open = false; /* full: closed, no tail */
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (tail_len) {
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Zeroed, so the error paths can free ->live and ->written. */
+++++++++++ ++ ++ ++++++++ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
+++++++++++ ++ ++ ++++++++ if (!new_run)
+++++++++++ ++ ++ ++++++++ return -ENOMEM;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Re-adoption (nocow only): after a remount (or eviction) the
+++++++++++ ++ ++ ++++++++ * inode's last partial stripe has no run, and a fresh claim would
+++++++++++ ++ ++ ++++++++ * abandon its free tail forever. When the allocation hint points
+++++++++++ ++ ++ ++++++++ * into a stripe wholly owned by this inode, adopt the tail as the
+++++++++++ ++ ++ ++++++++ * private run instead: every byte below the frontier is provably
+++++++++++ ++ ++ ++++++++ * the inode's own committed data, and nothing else can consume a
+++++++++++ ++ ++ ++++++++ * partial stripe's free space (see btrfs_claim_stripe_tail()).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_NOCOW && readopt_hint &&
+++++++++++ ++ ++ ++++++++ readopt_hint >= bg->start &&
+++++++++++ ++ ++ ++++++++ readopt_hint < bg->start + bg->length) {
+++++++++++ ++ ++ ++++++++ u64 stripe = bg->start +
+++++++++++ ++ ++ ++++++++ stripe_round_down(readopt_hint - bg->start, fsl);
+++++++++++ ++ ++ ++++++++ u64 frontier = 0;
+++++++++++ ++ ++ ++++++++ u64 tlen;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Under space_info->groups_sem (we are inside
+++++++++++ ++ ++ ++++++++ * find_free_extent()), so the lookup must not block -- see
+++++++++++ ++ ++ ++++++++ * stripe_extents_owned_by(). A trylock failure just declines
+++++++++++ ++ ++ ++++++++ * the re-adoption and falls through to a fresh claim below,
+++++++++++ ++ ++ ++++++++ * costing this stripe's free tail, not correctness.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (stripe_extents_owned_by(fs_info, root_id, ino, stripe, fsl,
+++++++++++ ++ ++ ++++++++ &frontier, true) &&
+++++++++++ ++ ++ ++++++++ frontier > stripe && frontier < stripe + fsl &&
+++++++++++ ++ ++ ++++++++ num_bytes <= stripe + fsl - frontier) {
+++++++++++ ++ ++ ++++++++ tlen = stripe + fsl - frontier;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * btrfs_stripe_run_range_usable() is for whole-stripe
+++++++++++ ++ ++ ++++++++ * claims (it rejects anything shorter than a full
+++++++++++ ++ ++ ++++++++ * stripe); a tail needs only the run-overlap check.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (stripe_range_run_free(bg, frontier, tlen) &&
+++++++++++ ++ ++ ++++++++ !btrfs_claim_stripe_tail(bg, frontier, tlen)) {
+++++++++++ ++ ++ ++++++++ start = frontier;
+++++++++++ ++ ++ ++++++++ len = tlen;
+++++++++++ ++ ++ ++++++++ goto install;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ ret = btrfs_claim_free_stripe_run(bg,
+++++++++++ ++ ++ ++++++++ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
+++++++++++ ++ ++ ++++++++ &start, &len);
+++++++++++ ++ ++ ++++++++ if (ret)
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ if (len < num_bytes) {
+++++++++++ ++ ++ ++++++++ /* Only a shorter run is free; see btrfs_alloc_from_open_stripe(). */
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++ ++ ++ ++++++++ *available = len;
+++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++install:
+++++++++++ ++ ++ ++++++++ /* Sleeps, so it has to happen before the locks below. */
+++++++++++ ++ ++ ++++++++ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
+++++++++++ ++ ++ ++++++++ new_run->written = new_run->live_nbits ?
+++++++++++ ++ ++ ++++++++ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* See the installation comment in btrfs_alloc_from_open_stripe(). */
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ if (list_empty(&bg->open_stripe_bg_list)) {
+++++++++++ ++ ++ ++++++++ btrfs_get_block_group(bg);
+++++++++++ ++ ++ ++++++++ list_add_tail(&bg->open_stripe_bg_list,
+++++++++++ ++ ++ ++++++++ &fs_info->open_stripe_bgs);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (bg->ro) {
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, start, len);
+++++++++++ ++ ++ ++++++++ *available = 0;
+++++++++++ ++ ++ ++++++++ ret = -ENOSPC;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ new_run->bg = bg;
+++++++++++ ++ ++ ++++++++ new_run->class = class;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * A NOCOW run outlives commits only while the stripe_alloc_allow_rmw
+++++++++++ ++ ++ ++++++++ * policy lets nocow data be written in place at all (see
+++++++++++ ++ ++ ++++++++ * close_block_group_stripe_runs()). Decided once, here: the commit
+++++++++++ ++ ++ ++++++++ * path's close and settle predicates must agree with each other for
+++++++++++ ++ ++ ++++++++ * the run's whole life, whatever the policy does meanwhile.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ new_run->persistent = class == BTRFS_STRIPE_RUN_NOCOW &&
+++++++++++ ++ ++ ++++++++ (btrfs_stripe_allow_rmw(fs_info) &
+++++++++++ ++ ++ ++++++++ (BTRFS_STRIPE_RMW_NODATACOW |
+++++++++++ ++ ++ ++++++++ BTRFS_STRIPE_RMW_PREALLOC));
+++++++++++ ++ ++ ++++++++ if (class == BTRFS_STRIPE_RUN_NOCOW)
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.nocow_run_opened);
+++++++++++ ++ ++ ++++++++ new_run->root = root_id;
+++++++++++ ++ ++ ++++++++ new_run->owner = ino;
+++++++++++ ++ ++ ++++++++ new_run->start = start;
+++++++++++ ++ ++ ++++++++ new_run->end = start + len;
+++++++++++ ++ ++ ++++++++ new_run->offset = start + num_bytes;
+++++++++++ ++ ++ ++++++++ new_run->inflight_bytes = num_bytes;
+++++++++++ ++ ++ ++++++++ new_run->open_seq = fs_info->stripe_retire_seq;
+++++++++++ ++ ++ ++++++++ new_run->open = (new_run->offset != new_run->end);
+++++++++++ ++ ++ ++++++++ run_live_set(new_run, start, num_bytes, true);
+++++++++++ ++ ++ ++++++++ list_add_tail(&new_run->list, &bg->open_stripe_runs);
+++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder += new_run->end - new_run->offset;
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ *ret_offset = start;
+++++++++++ ++ ++ ++++++++ new_run = NULL;
+++++++++++ ++ ++ ++++++++ ret = 0;
+++++++++++ ++ ++ ++++++++out:
+++++++++++ ++ ++ ++++++++ /* Unused on every path that did not install it, including errors. */
+++++++++++ ++ ++ ++++++++ if (new_run) {
+++++++++++ ++ ++ ++++++++ bitmap_free(new_run->live);
+++++++++++ ++ ++ ++++++++ bitmap_free(new_run->written);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ kfree(new_run);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * An extent inside a stripe run's range has been freed: its sectors no longer
+++++++++++ ++ ++ ++++++++ * hold data anyone intends to write, so clear their live bits.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * This is what makes a run detectably dead (open_stripe_run_dead()), so the
+++++++++++ ++ ++ ++++++++ * next commit can close it and release its stripes. Without it a nodatacow
+++++++++++ ++ ++ ++++++++ * or preallocating inode's private run hoards its stripes until the inode is
+++++++++++ ++ ++ ++++++++ * evicted. A partial write to one of those stripes can also pad over the
+++++++++++ ++ ++ ++++++++ * freed sectors now instead of falling back to a read-modify-write.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Takes only bg->stripe_run_lock, and the callers hold no free space lock
+++++++++++ ++ ++ ++++++++ * yet, so no nesting rule applies. The range is clamped to the run: a free
+++++++++++ ++ ++ ++++++++ * may span a run's boundary, and running off the live map would retire it.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_stripe_run_note_free(struct btrfs_block_group *bg, u64 bytenr,
+++++++++++ ++ ++ ++++++++ u64 size)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!btrfs_is_stripe_alloc_bg(bg))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ u64 start = max(bytenr, run->start);
+++++++++++ ++ ++ ++++++++ u64 end = min(bytenr + size, run->offset);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Only the persistent runs, which are the only ones that can
+++++++++++ ++ ++ ++++++++ * outlive their own data. Every other class is closed and
+++++++++++ ++ ++ ++++++++ * freed by the commit it belongs to, so its live map is never
+++++++++++ ++ ++ ++++++++ * consulted after its bytes could have been freed, and
+++++++++++ ++ ++ ++++++++ * clearing bits there would change padding decisions for no
+++++++++++ ++ ++ ++++++++ * gain here.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (!run->persistent)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (end <= start)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ run_live_set(run, start, end - start, false);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * A run's range must not be claimed again while the run object exists, even
+++++++++++ ++ ++ ++++++++ * if every byte of one of its stripes is back in the free space cache
+++++++++++ ++ ++ ++++++++ * (abandoned allocations): a second run over the same range would make the
+++++++++++ ++ ++ ++++++++ * range-to-run lookups (ordered extent attach, IO reporting) ambiguous and
+++++++++++ ++ ++ ++++++++ * corrupt the inflight accounting. Trim the candidate to end before the
+++++++++++ ++ ++ ++++++++ * first overlapping run, and reject it if its head overlaps or less than a
+++++++++++ ++ ++ ++++++++ * full stripe remains. Caller holds ctl->tree_lock; stripe_run_lock nests
+++++++++++ ++ ++ ++++++++ * inside it.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++bool btrfs_stripe_run_range_usable(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ u64 run_start, u64 *run_len)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ u64 len = *run_len;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (run->end <= run_start || run->start >= run_start + len)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (run->start <= run_start) {
+++++++++++ ++ ++ ++++++++ len = 0;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ len = run->start - run_start;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (len < bg->full_stripe_len) {
+++++++++++ ++ ++ ++++++++ atomic64_inc(&bg->fs_info->stripe_run_stats.claim_refused_live_run);
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ *run_len = len;
+++++++++++ ++ ++ ++++++++ return true;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * For the stripe_unusable scan: whole free stripes inside a live run's
+++++++++++ ++ ++ ++++++++ * range cannot be claimed while the run exists
+++++++++++ ++ ++ ++++++++ * (btrfs_stripe_run_range_usable()), so they must not count as claimable
+++++++++++ ++ ++ ++++++++ * or the admission gate would admit writes that cannot be placed. Clear
+++++++++++ ++ ++ ++++++++ * them from @freep and return their total; free_open_stripe_run() marks the
+++++++++++ ++ ++ ++++++++ * group for rescan so they are credited back when the run goes. Caller
+++++++++++ ++ ++ ++++++++ * holds ctl->tree_lock; stripe_run_lock nests inside it.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++u64 btrfs_stripe_run_stranded_stripes(struct btrfs_block_group *bg, u32 *freep,
+++++++++++ ++ ++ ++++++++ u64 nstripes)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ u64 stranded = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ u64 idx, last;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (run->end <= run->start)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ idx = div64_u64(run->start - bg->start, fsl);
+++++++++++ ++ ++ ++++++++ last = div64_u64(run->end - 1 - bg->start, fsl);
+++++++++++ ++ ++ ++++++++ for (; idx <= last && idx < nstripes; idx++) {
+++++++++++ ++ ++ ++++++++ if (freep[idx] != fsl)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ freep[idx] = 0;
+++++++++++ ++ ++ ++++++++ stranded += fsl;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (stranded)
+++++++++++ ++ ++ ++++++++ atomic64_add(stranded, &bg->fs_info->stripe_run_stats.strand_scan_bytes);
+++++++++++ ++ ++ ++++++++ return stranded;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Is the full stripe at @stripe inside a live run's range, i.e. stranded by
+++++++++++ ++ ++ ++++++++ * the rule of btrfs_stripe_run_stranded_stripes()? Caller holds
+++++++++++ ++ ++ ++++++++ * ctl->tree_lock.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++bool btrfs_stripe_run_strands(struct btrfs_block_group *bg, u64 stripe)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ bool ret = false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (run->end > run->start &&
+++++++++++ ++ ++ ++++++++ stripe < run->end && stripe + fsl > run->start) {
+++++++++++ ++ ++ ++++++++ ret = true;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Parse the value of the btrfs.stripe_alloc_allow_rmw property, a word list
+++++++++++ ++ ++ ++++++++ * ("nodatacow", "prealloc", "fsync"), into its mask. Words are separated by
+++++++++++ ++ ++ ++++++++ * comma, space or colon. An empty value yields an empty mask. Returns
+++++++++++ ++ ++ ++++++++ * -EINVAL on any unknown word.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++int btrfs_parse_stripe_rmw(const char *value, size_t len, u32 *mask)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ u32 m = 0;
+++++++++++ ++ ++ ++++++++ size_t i = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ while (i < len) {
+++++++++++ ++ ++ ++++++++ size_t j = i;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ while (j < len && value[j] != ',' && value[j] != ' ' &&
+++++++++++ ++ ++ ++++++++ value[j] != ':')
+++++++++++ ++ ++ ++++++++ j++;
+++++++++++ ++ ++ ++++++++ if (j - i == strlen("nodatacow") &&
+++++++++++ ++ ++ ++++++++ !strncmp(value + i, "nodatacow", j - i))
+++++++++++ ++ ++ ++++++++ m |= BTRFS_STRIPE_RMW_NODATACOW;
+++++++++++ ++ ++ ++++++++ else if (j - i == strlen("prealloc") &&
+++++++++++ ++ ++ ++++++++ !strncmp(value + i, "prealloc", j - i))
+++++++++++ ++ ++ ++++++++ m |= BTRFS_STRIPE_RMW_PREALLOC;
+++++++++++ ++ ++ ++++++++ else if (j - i == strlen("fsync") &&
+++++++++++ ++ ++ ++++++++ !strncmp(value + i, "fsync", j - i))
+++++++++++ ++ ++ ++++++++ m |= BTRFS_STRIPE_RMW_FSYNC;
+++++++++++ ++ ++ ++++++++ else if (j != i)
+++++++++++ ++ ++ ++++++++ return -EINVAL;
+++++++++++ ++ ++ ++++++++ i = j + 1;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ *mask = m;
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Does every committed extent in [start, start + len) belong solely to
+++++++++++ ++ ++ ++++++++ * @ino in @root_id -- exactly one reference, a plain data ref, count 1?
+++++++++++ ++ ++ ++++++++ * Any metadata block, shared reference, or foreign owner fails. Caller
+++++++++++ ++ ++ ++++++++ * limits the range to full stripes; uncommitted allocations need not be
+++++++++++ ++ ++ ++++++++ * visible because nothing can allocate into a partially used stripe, so a
+++++++++++ ++ ++ ++++++++ * foreign extent can only appear after the whole stripe frees and commits.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Callers holding space_info->groups_sem (anything reached from
+++++++++++ ++ ++ ++++++++ * find_free_extent()) must pass @nowait: delayed ref processing takes
+++++++++++ ++ ++ ++++++++ * extent tree locks and then groups_sem, so blocking on an extent tree lock
+++++++++++ ++ ++ ++++++++ * here can deadlock behind a queued groups_sem writer. Callers that must
+++++++++++ ++ ++ ++++++++ * not block for other reasons pass it too. With @nowait a contended lock
+++++++++++ ++ ++ ++++++++ * fails the search and the answer is false, which every caller treats only
+++++++++++ ++ ++ ++++++++ * as a reason to skip an optimisation.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static bool stripe_extents_owned_by(struct btrfs_fs_info *fs_info,
+++++++++++ ++ ++ ++++++++ u64 root_id, u64 ino, u64 start, u64 len,
+++++++++++ ++ ++ ++++++++ u64 *frontier, bool nowait)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_root *extent_root = btrfs_extent_root(fs_info, start);
+++++++++++ ++ ++ ++++++++ struct btrfs_path *path;
+++++++++++ ++ ++ ++++++++ struct btrfs_key key;
+++++++++++ ++ ++ ++++++++ bool ret = true;
+++++++++++ ++ ++ ++++++++ int err;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!extent_root)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ path = btrfs_alloc_path();
+++++++++++ ++ ++ ++++++++ if (!path)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Read-only search, so nowait is allowed, and every step of the walk
+++++++++++ ++ ++ ++++++++ * below honours it (btrfs_next_leaf() runs with time_seq 0).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ path->nowait = nowait;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ key.objectid = start;
+++++++++++ ++ ++ ++++++++ key.type = BTRFS_EXTENT_ITEM_KEY;
+++++++++++ ++ ++ ++++++++ key.offset = 0;
+++++++++++ ++ ++ ++++++++ err = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+++++++++++ ++ ++ ++++++++ if (err < 0) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ /* An extent starting before @start may still overlap it. */
+++++++++++ ++ ++ ++++++++ err = btrfs_previous_extent_item(extent_root, path, 0);
+++++++++++ ++ ++ ++++++++ if (err < 0) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (err > 0) {
+++++++++++ ++ ++ ++++++++ btrfs_release_path(path);
+++++++++++ ++ ++ ++++++++ key.objectid = start;
+++++++++++ ++ ++ ++++++++ key.type = BTRFS_EXTENT_ITEM_KEY;
+++++++++++ ++ ++ ++++++++ key.offset = 0;
+++++++++++ ++ ++ ++++++++ err = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+++++++++++ ++ ++ ++++++++ if (err < 0) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ while (true) {
+++++++++++ ++ ++ ++++++++ struct extent_buffer *leaf = path->nodes[0];
+++++++++++ ++ ++ ++++++++ int slot = path->slots[0];
+++++++++++ ++ ++ ++++++++ struct btrfs_extent_item *ei;
+++++++++++ ++ ++ ++++++++ struct btrfs_extent_inline_ref *iref;
+++++++++++ ++ ++ ++++++++ struct btrfs_extent_data_ref *dref;
+++++++++++ ++ ++ ++++++++ int type;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (slot >= btrfs_header_nritems(leaf)) {
+++++++++++ ++ ++ ++++++++ err = btrfs_next_leaf(extent_root, path);
+++++++++++ ++ ++ ++++++++ if (err < 0) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (err > 0)
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ btrfs_item_key_to_cpu(leaf, &key, slot);
+++++++++++ ++ ++ ++++++++ if (key.objectid >= start + len)
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ if (key.type == BTRFS_METADATA_ITEM_KEY) {
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * key.offset is the level, not a length; a tree
+++++++++++ ++ ++ ++++++++ * block spans nodesize bytes. One from before the
+++++++++++ ++ ++ ++++++++ * range (e.g. where the pre-range positioning landed
+++++++++++ ++ ++ ++++++++ * on a metadata block group's last item) is harmless.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (key.objectid + fs_info->nodesize > start) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ path->slots[0]++;
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (key.type != BTRFS_EXTENT_ITEM_KEY ||
+++++++++++ ++ ++ ++++++++ key.objectid + key.offset <= start) {
+++++++++++ ++ ++ ++++++++ path->slots[0]++;
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item);
+++++++++++ ++ ++ ++++++++ if (btrfs_extent_refs(leaf, ei) != 1 ||
+++++++++++ ++ ++ ++++++++ !(btrfs_extent_flags(leaf, ei) & BTRFS_EXTENT_FLAG_DATA)) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * With a keyed (non-inline) backref the item ends after the
+++++++++++ ++ ++ ++++++++ * header, and reading an inline ref would read the next item.
+++++++++++ ++ ++ ++++++++ * Treat it as not owned, the conservative answer.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (btrfs_item_size(leaf, slot) < sizeof(*ei) +
+++++++++++ ++ ++ ++++++++ btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY)) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ iref = (struct btrfs_extent_inline_ref *)(ei + 1);
+++++++++++ ++ ++ ++++++++ type = btrfs_get_extent_inline_ref_type(leaf, iref,
+++++++++++ ++ ++ ++++++++ BTRFS_REF_TYPE_DATA);
+++++++++++ ++ ++ ++++++++ if (type != BTRFS_EXTENT_DATA_REF_KEY) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ dref = (struct btrfs_extent_data_ref *)(&iref->offset);
+++++++++++ ++ ++ ++++++++ if (btrfs_extent_data_ref_root(leaf, dref) != root_id ||
+++++++++++ ++ ++ ++++++++ btrfs_extent_data_ref_objectid(leaf, dref) != ino ||
+++++++++++ ++ ++ ++++++++ btrfs_extent_data_ref_count(leaf, dref) != 1) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (frontier && key.objectid + key.offset > *frontier)
+++++++++++ ++ ++ ++++++++ *frontier = min(key.objectid + key.offset, start + len);
+++++++++++ ++ ++ ++++++++ path->slots[0]++;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++out:
+++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * May [start, start + num_bytes) be written in place under stripe-exclusive
+++++++++++ ++ ++ ++++++++ * allocation? Only when stripe_alloc_allow_rmw includes @which and every
+++++++++++ ++ ++ ++++++++ * full stripe the range touches is isolated to @inode: an in-place write
+++++++++++ ++ ++ ++++++++ * RMWs parity, so a crash can tear every extent sharing the stripe, and
+++++++++++ ++ ++ ++++++++ * isolation confines that to the file that opted out of COW. Fast path:
+++++++++++ ++ ++ ++++++++ * the stripe is in one of the inode's own NOCOW runs. Slow path: the
+++++++++++ ++ ++ ++++++++ * committed extent tree proves sole ownership. Extents that fail stay
+++++++++++ ++ ++ ++++++++ * COWed, and their rewrite lands in the inode's NOCOW run, so the next
+++++++++++ ++ ++ ++++++++ * overwrite passes.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++bool btrfs_stripe_nocow_writable(struct btrfs_inode *inode, u64 start,
+++++++++++ ++ ++ ++++++++ u64 num_bytes, u32 which, bool nowait)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = inode->root->fs_info;
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ const u64 root_id = btrfs_root_id(inode->root);
+++++++++++ ++ ++ ++++++++ const u64 ino = btrfs_ino(inode);
+++++++++++ ++ ++ ++++++++ u64 fsl;
+++++++++++ ++ ++ ++++++++ u64 cur;
+++++++++++ ++ ++ ++++++++ u64 end;
+++++++++++ ++ ++ ++++++++ bool ret = true;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!(btrfs_stripe_allow_rmw(fs_info) & which))
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, start);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ if (!btrfs_is_stripe_alloc_bg(bg)) {
+++++++++++ ++ ++ ++++++++ /* Not a raid56 data group: nothing forces COW here anyway. */
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ return true;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ fsl = bg->full_stripe_len;
+++++++++++ ++ ++ ++++++++ cur = bg->start + stripe_round_down(start - bg->start, fsl);
+++++++++++ ++ ++ ++++++++ end = start + num_bytes;
+++++++++++ ++ ++ ++++++++ while (cur < end) {
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ int verdict = 0; /* 0 = unknown, 1 = ok, -1 = foreign */
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (cur < run->start || cur >= run->end)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (run->class == BTRFS_STRIPE_RUN_NOCOW &&
+++++++++++ ++ ++ ++++++++ run->root == root_id && run->owner == ino)
+++++++++++ ++ ++ ++++++++ verdict = 1;
+++++++++++ ++ ++ ++++++++ else
+++++++++++ ++ ++ ++++++++ verdict = -1;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * No groups_sem here (this runs from the nocow check, not
+++++++++++ ++ ++ ++++++++ * from the allocator), so the reliable blocking search is
+++++++++++ ++ ++ ++++++++ * both safe and wanted: a spurious "not ours" would force
+++++++++++ ++ ++ ++++++++ * COW on an extent that does not need it. A NOWAIT write
+++++++++++ ++ ++ ++++++++ * must not block, so it gets the nonblocking search, and
+++++++++++ ++ ++ ++++++++ * COW (which it cannot do either) when that cannot answer.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (verdict == 0 &&
+++++++++++ ++ ++ ++++++++ !stripe_extents_owned_by(fs_info, root_id, ino, cur, fsl,
+++++++++++ ++ ++ ++++++++ NULL, nowait))
+++++++++++ ++ ++ ++++++++ verdict = -1;
+++++++++++ ++ ++ ++++++++ if (verdict < 0) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ cur += fsl;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Lower an allocation ceiling to the end of the last live sector at or above
+++++++++++ ++ ++ ++++++++ * @floor. Sectors between it and the frontier were allocated and abandoned:
+++++++++++ ++ ++ ++++++++ * they never get a bio, so a parked rbio counting on them would leave only
+++++++++++ ++ ++ ++++++++ * when kicked or at its park deadline. Caller holds bg->stripe_run_lock.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static u64 run_live_ceiling(const struct btrfs_open_stripe_run *run, u64 floor,
+++++++++++ ++ ++ ++++++++ u64 ceiling)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
+++++++++++ ++ ++ ++++++++ u32 first;
+++++++++++ ++ ++ ++++++++ u32 i;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!run->live || !run->live_nbits || floor < run->start ||
+++++++++++ ++ ++ ++++++++ ceiling <= floor)
+++++++++++ ++ ++ ++++++++ return ceiling;
+++++++++++ ++ ++ ++++++++ first = (floor - run->start) >> bits;
+++++++++++ ++ ++ ++++++++ i = (ceiling - run->start) >> bits;
+++++++++++ ++ ++ ++++++++ if (i > run->live_nbits)
+++++++++++ ++ ++ ++++++++ return ceiling;
+++++++++++ ++ ++ ++++++++ while (i > first && !test_bit(i - 1, run->live))
+++++++++++ ++ ++ ++++++++ i--;
+++++++++++ ++ ++ ++++++++ return run->start + ((u64)i << bits);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * How far has the covering run allocated into this stripe? Returns the
+++++++++++ ++ ++ ++++++++ * exclusive end of the allocated prefix, clamped to the stripe, or U64_MAX
+++++++++++ ++ ++ ++++++++ * when no run covers the stripe (nothing left to wait for; the caller
+++++++++++ ++ ++ ++++++++ * proceeds as if the stripe were fully allocated and fully arrived).
+++++++++++ ++ ++ ++++++++ * A partial write below the returned ceiling is waiting on neighbours whose
+++++++++++ ++ ++ ++++++++ * data IO is already in flight -- allocation happens at writeback submission
+++++++++++ ++ ++ ++++++++ * and the run's inflight accounting makes the commit wait for it -- so a
+++++++++++ ++ ++ ++++++++ * parked rbio covering [stripe_start, ceiling) can complete (padding the
+++++++++++ ++ ++ ++++++++ * dead tail), and one that does not should keep waiting for merges.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++u64 btrfs_stripe_run_alloc_ceiling(struct btrfs_fs_info *fs_info,
+++++++++++ ++ ++ ++++++++ u64 stripe_start, u64 stripe_len)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ u64 ceiling = U64_MAX;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++ ++ ++ ++++++++ return U64_MAX;
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ return U64_MAX;
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (stripe_start < run->start || stripe_start >= run->end)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ ceiling = clamp(run->offset, stripe_start,
+++++++++++ ++ ++ ++++++++ stripe_start + stripe_len);
+++++++++++ ++ ++ ++++++++ ceiling = run_live_ceiling(run, stripe_start, ceiling);
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ return ceiling;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Which sectors of this stripe may be padded?
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Fills @dead with one bit per sector, set where the sector holds nothing
+++++++++++ ++ ++ ++++++++ * anyone has written: never handed out, handed out and abandoned, or
+++++++++++ ++ ++ ++++++++ * handed out with the write not yet completed (per the run's written map).
+++++++++++ ++ ++ ++++++++ * The caller may zero-fill exactly those and write the whole stripe.
+++++++++++ ++ ++ ++++++++ * @unwritten, when non-NULL, gets the subset that is allocated but
+++++++++++ ++ ++ ++++++++ * unwritten.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Zeroing an allocated sector whose write is still to come is safe: the
+++++++++++ ++ ++ ++++++++ * caller holds the stripe lock, which every raid56 write to the stripe
+++++++++++ ++ ++ ++++++++ * takes, so that write lands after the caller's; completed writes are in
+++++++++++ ++ ++ ++++++++ * the written map before the stripe is unlocked (rbio_orig_end_io()); and
+++++++++++ ++ ++ ++++++++ * nothing reads the sector before its own write lands, since its extent is
+++++++++++ ++ ++ ++++++++ * not committed until then. A NOCOW run's written map is not consulted:
+++++++++++ ++ ++ ++++++++ * its live sectors are rewritten in place, so they may already hold data.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Returns false if no run covers the stripe; nothing may be padded then,
+++++++++++ ++ ++ ++++++++ * since an uncovered sector may hold committed data. @info, when non-NULL,
+++++++++++ ++ ++ ++++++++ * records the covering run's state for the pad-decline trace.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++bool btrfs_stripe_run_pad_mask(struct btrfs_fs_info *fs_info, u64 stripe_start,
+++++++++++ ++ ++ ++++++++ u64 stripe_len, unsigned long *dead,
+++++++++++ ++ ++ ++++++++ unsigned long *unwritten,
+++++++++++ ++ ++ ++++++++ struct btrfs_stripe_pad_info *info)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ const u32 bits = fs_info->sectorsize_bits;
+++++++++++ ++ ++ ++++++++ const u32 nsectors = stripe_len >> bits;
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ bool ret = false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (info) {
+++++++++++ ++ ++ ++++++++ memset(info, 0, sizeof(*info));
+++++++++++ ++ ++ ++++++++ info->reason = BTRFS_STRIPE_PAD_NO_RUN;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs)) {
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
+++++++++++ ++ ++ ++++++++ if (info)
+++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_BGS;
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+++++++++++ ++ ++ ++++++++ if (!bg) {
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
+++++++++++ ++ ++ ++++++++ if (info)
+++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_BG;
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (info)
+++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_COVER;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ u32 first;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (stripe_start < run->start || stripe_start >= run->end)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (info) {
+++++++++++ ++ ++ ++++++++ info->run_start = run->start;
+++++++++++ ++ ++ ++++++++ info->run_end = run->end;
+++++++++++ ++ ++ ++++++++ info->run_offset = run->offset;
+++++++++++ ++ ++ ++++++++ info->inflight_bytes = run->inflight_bytes;
+++++++++++ ++ ++ ++++++++ info->run_open = run->open;
+++++++++++ ++ ++ ++++++++ info->run_class = run->class;
+++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_NO_LIVE;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (!run->live || !run->live_nbits)
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * The stripe must lie inside the run's span. A closed run
+++++++++++ ++ ++ ++++++++ * has shrunk its end to the frontier, so its final stripe
+++++++++++ ++ ++ ++++++++ * overhangs: the sectors past end are the returned tail,
+++++++++++ ++ ++ ++++++++ * sub-stripe free space nothing can claim while the stripe
+++++++++++ ++ ++ ++++++++ * holds live data. They were never allocated, their live
+++++++++++ ++ ++ ++++++++ * bits are clear, and they are exactly what a kicked park
+++++++++++ ++ ++ ++++++++ * needs to pad. An open run's end is stripe aligned, so
+++++++++++ ++ ++ ++++++++ * overhang there means a stripe the run does not own.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (stripe_start + stripe_len > run->end && run->open) {
+++++++++++ ++ ++ ++++++++ if (info)
+++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_OVERHANG;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ first = (stripe_start - run->start) >> bits;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * And wholly inside the map. Declining here costs a
+++++++++++ ++ ++ ++++++++ * read-modify-write; reading past the map would be a lie about
+++++++++++ ++ ++ ++++++++ * which sectors are dead, and padding acts on that by
+++++++++++ ++ ++ ++++++++ * overwriting them.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (first + nsectors > run->live_nbits) {
+++++++++++ ++ ++ ++++++++ if (info)
+++++++++++ ++ ++ ++++++++ info->norun = BTRFS_STRIPE_NORUN_PAST_MAP;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ bitmap_zero(dead, nsectors);
+++++++++++ ++ ++ ++++++++ if (unwritten)
+++++++++++ ++ ++ ++++++++ bitmap_zero(unwritten, nsectors);
+++++++++++ ++ ++ ++++++++ for (u32 i = 0; i < nsectors; i++) {
+++++++++++ ++ ++ ++++++++ if (!test_bit(first + i, run->live)) {
+++++++++++ ++ ++ ++++++++ set_bit(i, dead);
+++++++++++ ++ ++ ++++++++ } else if (run->written &&
+++++++++++ ++ ++ ++++++++ run->class != BTRFS_STRIPE_RUN_NOCOW &&
+++++++++++ ++ ++ ++++++++ !test_bit(first + i, run->written)) {
+++++++++++ ++ ++ ++++++++ set_bit(i, dead);
+++++++++++ ++ ++ ++++++++ if (unwritten)
+++++++++++ ++ ++ ++++++++ set_bit(i, unwritten);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (info)
+++++++++++ ++ ++ ++++++++ info->reason = BTRFS_STRIPE_PAD_OK;
+++++++++++ ++ ++ ++++++++ ret = true;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (!ret)
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/* No open or draining run covers @bytenr any more. */
+++++++++++ ++ ++ ++++++++static bool stripe_log_range_settled(struct btrfs_block_group *bg, u64 bytenr)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ bool settled = true;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (bytenr >= run->start && bytenr < run->offset) {
+++++++++++ ++ ++ ++++++++ settled = false;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ return settled;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Remember, on the logging inode, a logged extent that lives in the final
+++++++++++ ++ ++ ++++++++ * partial stripe of one of the inode's recently closed runs, so that the
+++++++++++ ++ ++ ++++++++ * next fsync can re-dirty its file range and carry the data forward into
+++++++++++ ++ ++ ++++++++ * the then-current run -- letting the old partial stripe free whole. A
+++++++++++ ++ ++ ++++++++ * zero @file_len means the caller does not want this extent carried
+++++++++++ ++ ++ ++++++++ * (e.g. compressed extents, which would recompress). @ps_len is nonzero
+++++++++++ ++ ++ ++++++++ * when the settling walk could see the run and computed its final partial
+++++++++++ ++ ++ ++++++++ * stripe. Every table here is small and bounded; on overflow the extent
+++++++++++ ++ ++ ++++++++ * simply is not carried and its stripe frees the slow way. Caller holds
+++++++++++ ++ ++ ++++++++ * the inode's log_mutex, which serializes against the fsync-entry consumer.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void log_carry_record(struct btrfs_inode *inode, u64 file_start,
+++++++++++ ++ ++ ++++++++ u64 file_len, u64 bytenr, u64 num_bytes,
+++++++++++ ++ ++ ++++++++ u64 ps_start, u64 ps_len)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_log_carry *carry = inode->log_carry;
+++++++++++ ++ ++ ++++++++ int n, i;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!carry) {
+++++++++++ ++ ++ ++++++++ carry = kzalloc(sizeof(*carry), GFP_NOFS);
+++++++++++ ++ ++ ++++++++ if (!carry)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ inode->log_carry = carry;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (ps_len) {
+++++++++++ ++ ++ ++++++++ n = min_t(int, carry->nr_ps, BTRFS_LOG_CARRY_NR_PS);
+++++++++++ ++ ++ ++++++++ for (i = 0; i < n; i++)
+++++++++++ ++ ++ ++++++++ if (carry->ps_start[i] == ps_start)
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ if (i == n) {
+++++++++++ ++ ++ ++++++++ int slot = carry->nr_ps % BTRFS_LOG_CARRY_NR_PS;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ carry->ps_start[slot] = ps_start;
+++++++++++ ++ ++ ++++++++ carry->ps_len[slot] = ps_len;
+++++++++++ ++ ++ ++++++++ carry->nr_ps++;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (!file_len || carry->nr_ranges >= BTRFS_LOG_CARRY_NR_RANGES)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ n = min_t(int, carry->nr_ps, BTRFS_LOG_CARRY_NR_PS);
+++++++++++ ++ ++ ++++++++ for (i = 0; i < n; i++)
+++++++++++ ++ ++ ++++++++ if (bytenr < carry->ps_start[i] + carry->ps_len[i] &&
+++++++++++ ++ ++ ++++++++ bytenr + num_bytes > carry->ps_start[i])
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ if (i == n)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ for (i = 0; i < carry->nr_ranges; i++)
+++++++++++ ++ ++ ++++++++ if (carry->range[i].start == file_start &&
+++++++++++ ++ ++ ++++++++ carry->range[i].len == file_len)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ carry->range[carry->nr_ranges].start = file_start;
+++++++++++ ++ ++ ++++++++ carry->range[carry->nr_ranges].len = file_len;
+++++++++++ ++ ++ ++++++++ carry->range[carry->nr_ranges].disk_bytenr = bytenr;
+++++++++++ ++ ++ ++++++++ carry->nr_ranges++;
+++++++++++ ++ ++ ++++++++ atomic64_inc(&inode->root->fs_info->stripe_run_stats.log_carry_ranges);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Disk end of @inode's last regular file extent, or 0: the re-adoption
+++++++++++ ++ ++ ++++++++ * candidate for a nocow allocation arriving with no allocation hint --
+++++++++++ ++ ++ ++++++++ * after a remount or eviction the extent maps are gone, so the normal
+++++++++++ ++ ++ ++++++++ * em-based hint is empty exactly when re-adoption matters most.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++u64 btrfs_nocow_alloc_hint(struct btrfs_inode *inode)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_root *root = inode->root;
+++++++++++ ++ ++ ++++++++ struct btrfs_path *path;
+++++++++++ ++ ++ ++++++++ struct btrfs_key key;
+++++++++++ ++ ++ ++++++++ struct btrfs_file_extent_item *fi;
+++++++++++ ++ ++ ++++++++ u64 hint = 0;
+++++++++++ ++ ++ ++++++++ int ret;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ path = btrfs_alloc_path();
+++++++++++ ++ ++ ++++++++ if (!path)
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++ key.objectid = btrfs_ino(inode);
+++++++++++ ++ ++ ++++++++ key.type = BTRFS_EXTENT_DATA_KEY;
+++++++++++ ++ ++ ++++++++ key.offset = (u64)-1;
+++++++++++ ++ ++ ++++++++ ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
+++++++++++ ++ ++ ++++++++ if (ret < 0)
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ ret = btrfs_previous_item(root, path, key.objectid,
+++++++++++ ++ ++ ++++++++ BTRFS_EXTENT_DATA_KEY);
+++++++++++ ++ ++ ++++++++ if (ret)
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
+++++++++++ ++ ++ ++++++++ fi = btrfs_item_ptr(path->nodes[0], path->slots[0],
+++++++++++ ++ ++ ++++++++ struct btrfs_file_extent_item);
+++++++++++ ++ ++ ++++++++ if (btrfs_file_extent_type(path->nodes[0], fi) ==
+++++++++++ ++ ++ ++++++++ BTRFS_FILE_EXTENT_INLINE)
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ if (btrfs_file_extent_disk_bytenr(path->nodes[0], fi))
+++++++++++ ++ ++ ++++++++ hint = btrfs_file_extent_disk_bytenr(path->nodes[0], fi) +
+++++++++++ ++ ++ ++++++++ btrfs_file_extent_disk_num_bytes(path->nodes[0], fi);
+++++++++++ ++ ++ ++++++++out:
+++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
+++++++++++ ++ ++ ++++++++ return hint;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/* Does no run object (open or draining) overlap [start, start + len)? */
+++++++++++ ++ ++ ++++++++static bool stripe_range_run_free(struct btrfs_block_group *bg, u64 start,
+++++++++++ ++ ++ ++++++++ u64 len)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ bool ret = true;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (run->start < start + len && run->end > start) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/* Does any stripe run, open or draining, still cover part of @bg? */
+++++++++++ ++ ++ ++++++++bool btrfs_block_group_has_stripe_runs(struct btrfs_block_group *bg)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ bool ret;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ ret = !list_empty(&bg->open_stripe_runs);
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Close @inode's open LOG and NOCOW runs at eviction, so a persistent NOCOW
+++++++++++ ++ ++ ++++++++ * run's claimed tail does not stay unusable for the rest of the mount. The
+++++++++++ ++ ++ ++++++++ * tails return to the free space cache.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_close_inode_stripe_runs(struct btrfs_inode *inode)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = inode->root->fs_info;
+++++++++++ ++ ++ ++++++++ const u64 root_id = btrfs_root_id(inode->root);
+++++++++++ ++ ++ ++++++++ const u64 ino = btrfs_ino(inode);
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++restart:
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs,
+++++++++++ ++ ++ ++++++++ open_stripe_bg_list) {
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ u64 tail_start = 0;
+++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (!run->open || run->root != root_id ||
+++++++++++ ++ ++ ++++++++ run->owner != ino ||
+++++++++++ ++ ++ ++++++++ (run->class != BTRFS_STRIPE_RUN_LOG &&
+++++++++++ ++ ++ ++++++++ run->class != BTRFS_STRIPE_RUN_NOCOW))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (tail_len) {
+++++++++++ ++ ++ ++++++++ btrfs_get_block_group(bg);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ goto restart;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * A log commit on a shared LOG run: instead of closing the run, move its
+++++++++++ ++ ++ ++++++++ * frontier past the stripe holding the end of the logged extent, so nothing
+++++++++++ ++ ++ ++++++++ * allocates beside the extent after the log commit while the run goes on in
+++++++++++ ++ ++ ++++++++ * the next stripe. The skipped sectors are not live (the pad oracle fills
+++++++++++ ++ ++ ++++++++ * them) and go back to the free space cache like a closed run's tail: inside
+++++++++++ ++ ++ ++++++++ * a partly live stripe they count as trapped until the stripe frees, and a
+++++++++++ ++ ++ ++++++++ * later close returns only [offset, end), which no longer includes them.
+++++++++++ ++ ++ ++++++++ * Caller holds bg->stripe_run_lock. Returns the skipped length.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static u64 seal_log_stripe(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run, u64 ext_end,
+++++++++++ ++ ++ ++++++++ u64 *skip_start)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++ ++ ++ ++++++++ u64 boundary;
+++++++++++ ++ ++ ++++++++ u64 skip;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!run->open)
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++ boundary = bg->start + stripe_round_down(ext_end - 1 - bg->start, fsl) + fsl;
+++++++++++ ++ ++ ++++++++ if (run->offset >= boundary)
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++ boundary = min(boundary, run->end);
+++++++++++ ++ ++ ++++++++ skip = boundary - run->offset;
+++++++++++ ++ ++ ++++++++ *skip_start = run->offset;
+++++++++++ ++ ++ ++++++++ run->offset = boundary;
+++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder -= skip;
+++++++++++ ++ ++ ++++++++ if (run->offset == run->end) {
+++++++++++ ++ ++ ++++++++ int class;
+++++++++++ ++ ++ ++++++++ int band;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
+++++++++++ ++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+++++++++++ ++ ++ ++++++++ if (bg->open_stripe[class][band] == run)
+++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++ ++ ++ ++++++++ run->open = false;
+++++++++++ ++ ++ ++++++++ if (open_stripe_run_drained(run) && !run->retained)
+++++++++++ ++ ++ ++++++++ free_open_stripe_run(bg, run);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ return skip;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++static bool run_unwritten_in(struct btrfs_open_stripe_run *run, u64 start,
+++++++++++ ++ ++ ++++++++ u64 len);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Per-stripe settle (stripe_log_data_shared): every allocated sector of the
+++++++++++ ++ ++ ++++++++ * stripes holding [bytenr, bytenr + num_bytes) has been written, or no run
+++++++++++ ++ ++ ++++++++ * covers them any more. Writers sharing those stripes park and merge into
+++++++++++ ++ ++ ++++++++ * the same full-stripe write, so this waits for them and for nothing else in
+++++++++++ ++ ++ ++++++++ * the run.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static bool stripe_log_extent_written(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ u64 bytenr, u64 num_bytes)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++ ++ ++ ++++++++ const u64 s = bg->start + stripe_round_down(bytenr - bg->start, fsl);
+++++++++++ ++ ++ ++++++++ const u64 e = bg->start +
+++++++++++ ++ ++ ++++++++ stripe_round_down(bytenr + num_bytes - bg->start + fsl - 1, fsl);
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ bool written = true;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (bytenr < run->start || bytenr >= run->offset)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (run_unwritten_in(run, s, e - s))
+++++++++++ ++ ++ ++++++++ written = false;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ return written;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * A log commit is about to make the extent at [bytenr, bytenr + num_bytes)
+++++++++++ ++ ++ ++++++++ * durable, after which no later write may extend or RMW its stripes. Close
+++++++++++ ++ ++ ++++++++ * the run covering the extent so nothing more allocates into its stripes,
+++++++++++ ++ ++ ++++++++ * kick any parked partial writes there, and wait for all IO into the run to
+++++++++++ ++ ++ ++++++++ * finish. This is a pure data-IO wait (no transaction joins), bounded by
+++++++++++ ++ ++ ++++++++ * bio flight time because parked writes are flushed first, and is safe
+++++++++++ ++ ++ ++++++++ * under the inode log mutex and a running transaction handle.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * With stripe_log_data_shared a shared LOG run is not closed: the stripe
+++++++++++ ++ ++ ++++++++ * holding the end of the extent is sealed (seal_log_stripe()). In that
+++++++++++ ++ ++ ++++++++ * mode the wait, for any run, is for every allocated sector of the
+++++++++++ ++ ++ ++++++++ * extent's stripes to be written (stripe_log_extent_written()) rather than
+++++++++++ ++ ++ ++++++++ * for the run to drain. Neither wait has a timeout: returning early would
+++++++++++ ++ ++ ++++++++ * let a neighbour's partial write RMW the extent after it is durable.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * [file_start, file_start + file_len) is the extent's file range, recorded
+++++++++++ ++ ++ ++++++++ * for carry-forward when the extent is in the run's final partial stripe
+++++++++++ ++ ++ ++++++++ * (stripe_log_data_shared clear only); file_len == 0 opts out of carrying.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_log_settle_stripes(struct btrfs_inode *inode, u64 file_start,
+++++++++++ ++ ++ ++++++++ u64 file_len, u64 bytenr, u64 num_bytes)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = inode->root->fs_info;
+++++++++++ ++ ++ ++++++++ /* Once: the seal and the wait below must agree on the mode. */
+++++++++++ ++ ++ ++++++++ const bool shared = READ_ONCE(fs_info->stripe_log_data_shared);
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ u64 tail_start = 0;
+++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
+++++++++++ ++ ++ ++++++++ u64 flush_start = 0;
+++++++++++ ++ ++ ++++++++ u64 flush_len = 0;
+++++++++++ ++ ++ ++++++++ u64 ps_start = 0;
+++++++++++ ++ ++ ++++++++ u64 ps_len = 0;
+++++++++++ ++ ++ ++++++++ u64 seal_start = 0;
+++++++++++ ++ ++ ++++++++ u64 seal_len = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* stripe_alloc_allow_rmw=fsync waives the log-commit guarantee. */
+++++++++++ ++ ++ ++++++++ if (btrfs_stripe_allow_rmw(fs_info) & BTRFS_STRIPE_RMW_FSYNC)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ u64 filled;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (bytenr < run->start || bytenr >= run->offset)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * A nodatacow file's logged extents get no fsync survival
+++++++++++ ++ ++ ++++++++ * guarantee -- the file's own later in-place writes can
+++++++++++ ++ ++ ++++++++ * always tear them -- so closing its run would trap the
+++++++++++ ++ ++ ++++++++ * tail for nothing.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (run->class == BTRFS_STRIPE_RUN_NOCOW)
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ if (shared && (run->owner & BIT_ULL(63))) {
+++++++++++ ++ ++ ++++++++ const u64 fsl = bg->full_stripe_len;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* only the logged extent's stripes, not the shared run */
+++++++++++ ++ ++ ++++++++ flush_start = bg->start + stripe_round_down(bytenr - bg->start, fsl);
+++++++++++ ++ ++ ++++++++ flush_len = bg->start + stripe_round_down(bytenr + num_bytes - 1 - bg->start, fsl) +
+++++++++++ ++ ++ ++++++++ fsl - flush_start;
+++++++++++ ++ ++ ++++++++ seal_len = seal_log_stripe(bg, run, bytenr + num_bytes, &seal_start);
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ flush_start = run->start;
+++++++++++ ++ ++ ++++++++ flush_len = run->end - run->start;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * The run's final partial stripe, whether this call closes
+++++++++++ ++ ++ ++++++++ * the run or an earlier settle already did (offset froze at
+++++++++++ ++ ++ ++++++++ * close). Runs start stripe-aligned, so alignment relative
+++++++++++ ++ ++ ++++++++ * to run->start is stripe alignment.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ filled = run->offset - run->start;
+++++++++++ ++ ++ ++++++++ if (!stripe_is_aligned(filled, bg->full_stripe_len)) {
+++++++++++ ++ ++ ++++++++ ps_start = run->start +
+++++++++++ ++ ++ ++++++++ stripe_round_down(filled, bg->full_stripe_len);
+++++++++++ ++ ++ ++++++++ ps_len = bg->full_stripe_len;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (run->open) {
+++++++++++ ++ ++ ++++++++ int class;
+++++++++++ ++ ++ ++++++++ int band;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
+++++++++++ ++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+++++++++++ ++ ++ ++++++++ if (bg->open_stripe[class][band] == run)
+++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!shared)
+++++++++++ ++ ++ ++++++++ log_carry_record(inode, file_start, file_len, bytenr, num_bytes,
+++++++++++ ++ ++ ++++++++ ps_start, ps_len);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (seal_len) {
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, seal_start, seal_len);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.log_seals);
+++++++++++ ++ ++ ++++++++ atomic64_add(seal_len, &fs_info->stripe_run_stats.log_seal_bytes);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (tail_len) {
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (flush_len) {
+++++++++++ ++ ++ ++++++++ ktime_t t0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, flush_start, flush_len, false);
+++++++++++ ++ ++ ++++++++ t0 = ktime_get();
+++++++++++ ++ ++ ++++++++ if (shared)
+++++++++++ ++ ++ ++++++++ wait_var_event(&bg->open_stripe_runs,
+++++++++++ ++ ++ ++++++++ stripe_log_extent_written(bg, bytenr, num_bytes) ||
+++++++++++ ++ ++ ++++++++ BTRFS_FS_ERROR(fs_info));
+++++++++++ ++ ++ ++++++++ else
+++++++++++ ++ ++ ++++++++ wait_var_event(&bg->open_stripe_runs,
+++++++++++ ++ ++ ++++++++ stripe_log_range_settled(bg, bytenr) ||
+++++++++++ ++ ++ ++++++++ BTRFS_FS_ERROR(fs_info));
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.log_settle_waits);
+++++++++++ ++ ++ ++++++++ atomic64_add(ktime_to_ns(ktime_sub(ktime_get(), t0)),
+++++++++++ ++ ++ ++++++++ &fs_info->stripe_run_stats.log_settle_wait_ns);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Can stripe-exclusive allocation be enabled on this filesystem? Shared by
+++++++++++ ++ ++ ++++++++ * the mount option validation and the "stripe_alloc" filesystem property.
+++++++++++ ++ ++ ++++++++ * Called on a mounted (or mounting, options finalized) filesystem.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++int btrfs_stripe_alloc_check_support(struct btrfs_fs_info *fs_info)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Mirror btrfs_check_mountopts(): only the v1 space cache is a
+++++++++++ ++ ++ ++++++++ * problem. Its cache inode is nodatacow and preallocated, so a
+++++++++++ ++ ++ ++++++++ * commit overwrites it in place -- a sub-stripe write into committed
+++++++++++ ++ ++ ++++++++ * stripes that no csum would show afterwards. The free space tree
+++++++++++ ++ ++ ++++++++ * and no cache at all are both fine; nothing here reads either one.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Cache inodes left on disk by an earlier v1 mount do not matter:
+++++++++++ ++ ++ ++++++++ * only cache_save_setup() writes them, and only with SPACE_CACHE.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (btrfs_test_opt(fs_info, SPACE_CACHE)) {
+++++++++++ ++ ++ ++++++++ btrfs_err(fs_info,
+++++++++++ ++ ++ ++++++++ "stripe_alloc is not supported with space_cache=v1");
+++++++++++ ++ ++ ++++++++ return -EINVAL;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (btrfs_is_zoned(fs_info)) {
+++++++++++ ++ ++ ++++++++ btrfs_err(fs_info,
+++++++++++ ++ ++ ++++++++ "stripe_alloc is not supported on zoned filesystems");
+++++++++++ ++ ++ ++++++++ return -EINVAL;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Mixed block groups are refused for the mount option in
+++++++++++ ++ ++ ++++++++ * btrfs_check_mountopts(); refuse them here too, or the property
+++++++++++ ++ ++ ++++++++ * becomes a way to enable the policy that skips that check. A
+++++++++++ ++ ++ ++++++++ * mixed group is excluded from the claim rule anyway
+++++++++++ ++ ++ ++++++++ * (btrfs_is_stripe_alloc_bg() requires DATA without METADATA), so
+++++++++++ ++ ++ ++++++++ * enabling here would protect nothing while reporting success.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (btrfs_fs_incompat(fs_info, MIXED_GROUPS)) {
+++++++++++ ++ ++ ++++++++ btrfs_err(fs_info,
+++++++++++ ++ ++ ++++++++ "stripe_alloc is not supported on mixed block groups");
+++++++++++ ++ ++ ++++++++ return -EINVAL;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Arm or disarm stripe_unusable accounting across all raid56 data and
+++++++++++ ++ ++ ++++++++ * metadata block groups whose caches are loaded. Same walk as the
+++++++++++ ++ ++ ++++++++ * commit-time rescan.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void stripe_alloc_sweep_groups(struct btrfs_fs_info *fs_info, bool arm)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_space_info *sinfo;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ list_for_each_entry(sinfo, &fs_info->space_info, list) {
+++++++++++ ++ ++ ++++++++ int raid;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!(sinfo->flags & (BTRFS_BLOCK_GROUP_DATA |
+++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_METADATA |
+++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Metadata: the scan that maintains the stripe_meta margin
+++++++++++ ++ ++ ++++++++ * and reserve stops with the option, so clear them here or
+++++++++++ ++ ++ ++++++++ * they hold metadata back forever. The groups themselves
+++++++++++ ++ ++ ++++++++ * are armed and disarmed below like data groups: raid56
+++++++++++ ++ ++ ++++++++ * metadata places tree blocks in whole stripes from the
+++++++++++ ++ ++ ++++++++ * moment the option is on, so its trapped and claimable
+++++++++++ ++ ++ ++++++++ * bytes count from that moment and stop counting when it
+++++++++++ ++ ++ ++++++++ * goes off.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (!arm && (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA)) {
+++++++++++ ++ ++ ++++++++ spin_lock(&sinfo->lock);
+++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_margin = 0;
+++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_reserve = 0;
+++++++++++ ++ ++ ++++++++ btrfs_try_granting_tickets(sinfo);
+++++++++++ ++ ++ ++++++++ spin_unlock(&sinfo->lock);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ down_read(&sinfo->groups_sem);
+++++++++++ ++ ++ ++++++++ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!(btrfs_raid_array[raid].bg_flag &
+++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_RAID56_MASK))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &sinfo->block_groups[raid],
+++++++++++ ++ ++ ++++++++ list) {
+++++++++++ ++ ++ ++++++++ if (!arm)
+++++++++++ ++ ++ ++++++++ btrfs_block_group_disarm_stripe_unusable(bg);
+++++++++++ ++ ++ ++++++++ else if (bg->cached == BTRFS_CACHE_FINISHED)
+++++++++++ ++ ++ ++++++++ btrfs_block_group_init_stripe_unusable(bg);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ up_read(&sinfo->groups_sem);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Drain the writes the legacy allocator placed before stripe_alloc was
+++++++++++ ++ ++ ++++++++ * enabled on a live filesystem, then arm the write-hole invariant check.
+++++++++++ ++ ++ ++++++++ * Runs in a worker: the property that enables the policy is applied inside
+++++++++++ ++ ++ ++++++++ * a transaction, where flushing delalloc cannot wait.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_stripe_alloc_enable_work(struct work_struct *work)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = container_of(work, struct btrfs_fs_info,
+++++++++++ ++ ++ ++++++++ stripe_alloc_enable_work);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ btrfs_start_delalloc_roots(fs_info, LONG_MAX, false);
+++++++++++ ++ ++ ++++++++ btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Tree blocks the legacy allocator placed in the transaction running at
+++++++++++ ++ ++ ++++++++ * the flip are written into partly used stripes by that transaction's
+++++++++++ ++ ++ ++++++++ * commit. Wait for it before arming the check, which would otherwise
+++++++++++ ++ ++ ++++++++ * report those writes as violations.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ btrfs_commit_current_transaction(fs_info->tree_root);
+++++++++++ ++ ++ ++++++++ clear_bit(BTRFS_FS_STRIPE_ALLOC_ENABLING, &fs_info->flags);
+++++++++++ ++ ++ ++++++++ btrfs_info(fs_info,
+++++++++++ ++ ++ ++++++++ "stripe_alloc: writes placed before the policy was enabled have landed");
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Turn on stripe-exclusive allocation, from the "stripe_alloc" filesystem
+++++++++++ ++ ++ ++++++++ * property on the top-level subvolume's root directory. Runs during mount
+++++++++++ ++ ++ ++++++++ * when the root directory inode loads its properties -- before any user IO
+++++++++++ ++ ++ ++++++++ * -- or at runtime from btrfs_set_prop().
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++int btrfs_enable_stripe_alloc(struct btrfs_fs_info *fs_info)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ bool live = test_bit(BTRFS_FS_OPEN, &fs_info->flags);
+++++++++++ ++ ++ ++++++++ int ret;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++ ret = btrfs_stripe_alloc_check_support(fs_info);
+++++++++++ ++ ++ ++++++++ if (ret)
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * On a live filesystem the legacy allocator may still have writes in
+++++++++++ ++ ++ ++++++++ * flight. Their extents sit in partly used stripes that no stripe run
+++++++++++ ++ ++ ++++++++ * covers, so the write-hole invariant check would report them although
+++++++++++ ++ ++ ++++++++ * they were placed before the policy existed. Keep the check off until
+++++++++++ ++ ++ ++++++++ * every ordered extent that could have been allocated before the flip
+++++++++++ ++ ++ ++++++++ * has landed; extents allocated after it have stripe runs. A mount-time
+++++++++++ ++ ++ ++++++++ * enable has nothing in flight and skips the drain.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (live)
+++++++++++ ++ ++ ++++++++ set_bit(BTRFS_FS_STRIPE_ALLOC_ENABLING, &fs_info->flags);
+++++++++++ ++ ++ ++++++++ btrfs_set_opt(fs_info->mount_opt, STRIPE_ALLOC);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Arm stripe_unusable accounting for raid56 data groups whose caches
+++++++++++ ++ ++ ++++++++ * loaded before the policy was enabled: block groups are read before
+++++++++++ ++ ++ ++++++++ * the root directory's properties apply during mount, and a runtime
+++++++++++ ++ ++ ++++++++ * property set can arrive at any point. Groups still caching arm
+++++++++++ ++ ++ ++++++++ * when their caching thread finishes, now that the policy is on.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ stripe_alloc_sweep_groups(fs_info, true);
+++++++++++ ++ ++ ++++++++ /* At mount, btrfs_fill_super() reports the outcome. */
+++++++++++ ++ ++ ++++++++ if (live) {
+++++++++++ ++ ++ ++++++++ btrfs_info(fs_info,
+++++++++++ ++ ++ ++++++++ "using stripe-exclusive allocation for raid56 data (filesystem property)");
+++++++++++ ++ ++ ++++++++ queue_work(system_unbound_wq, &fs_info->stripe_alloc_enable_work);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Turn off stripe-exclusive allocation (property deleted). Open runs are
+++++++++++ ++ ++ ++++++++ * not touched here: the commit-time retirement runs unconditionally and
+++++++++++ ++ ++ ++++++++ * drains them at the next transaction commit.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_disable_stripe_alloc(struct btrfs_fs_info *fs_info)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ /* Disarm while the policy bit still passes the per-group gate. */
+++++++++++ ++ ++ ++++++++ stripe_alloc_sweep_groups(fs_info, false);
+++++++++++ ++ ++ ++++++++ btrfs_clear_opt(fs_info->mount_opt, STRIPE_ALLOC);
+++++++++++ ++ ++ ++++++++ btrfs_info(fs_info,
+++++++++++ ++ ++ ++++++++ "stripe-exclusive allocation for raid56 data disabled");
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Does @logical lie within a stripe run, open or draining? The raid56 layer
+++++++++++ ++ ++ ++++++++ * uses this to decide whether a partial write may be parked to merge into a
+++++++++++ ++ ++ ++++++++ * full-stripe write. An open run fills the rest of the stripe later or
+++++++++++ ++ ++ ++++++++ * never; a draining run's frontier is frozen, so everything below it is IO
+++++++++++ ++ ++ ++++++++ * the commit drain already waits for and everything above it is dead space
+++++++++++ ++ ++ ++++++++ * padding may fill.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++bool btrfs_stripe_in_open_run(struct btrfs_fs_info *fs_info, u64 logical)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ return btrfs_stripe_open_run_class(fs_info, logical, NULL);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * The part of [*start, *start + *len) that @run owns and tracks in its
+++++++++++ ++ ++ ++++++++ * written map: inside [run->start, run->end) -- the map itself reaches
+++++++++++ ++ ++ ++++++++ * further (it spans at least BTRFS_STRIPE_RUN_LIVE_STRIPES stripes from
+++++++++++ ++ ++ ++++++++ * run->start, past run->end into later runs' ranges), so it must never be
+++++++++++ ++ ++ ++++++++ * consulted outside the run's own range -- and inside the map. Nodatacow
+++++++++++ ++ ++ ++++++++ * runs keep no written map: they are rewritten in place for as long as the
+++++++++++ ++ ++ ++++++++ * inode lives and are never complete. Caller holds stripe_run_lock.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static bool run_written_range(const struct btrfs_open_stripe_run *run,
+++++++++++ ++ ++ ++++++++ u64 *start, u64 *len)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
+++++++++++ ++ ++ ++++++++ u64 lo, hi;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (run->class == BTRFS_STRIPE_RUN_NOCOW)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ if (!run->live || !run->written || !run->live_nbits)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ lo = max(*start, run->start);
+++++++++++ ++ ++ ++++++++ hi = min3(*start + *len, run->end,
+++++++++++ ++ ++ ++++++++ run->start + ((u64)run->live_nbits << bits));
+++++++++++ ++ ++ ++++++++ if (lo >= hi)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ *start = lo;
+++++++++++ ++ ++ ++++++++ *len = hi - lo;
+++++++++++ ++ ++ ++++++++ return true;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * A raid56 write of [start, start + len) has completed: mark it written in
+++++++++++ ++ ++ ++++++++ * the run that owns it. Sectors that are not live in that run (freed while
+++++++++++ ++ ++ ++++++++ * the write was in flight, or a bookkeeping bug) are counted in
+++++++++++ ++ ++ ++++++++ * written_not_live: the pad oracle trusts this map to say which allocated
+++++++++++ ++ ++ ++++++++ * sectors hold data, so a write it misses could be zeroed.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_stripe_run_mark_written(struct btrfs_fs_info *fs_info, u64 start,
+++++++++++ ++ ++ ++++++++ u64 len)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ const u32 bits = fs_info->sectorsize_bits;
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ u32 not_live = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, start);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ u64 s = start, l = len;
+++++++++++ ++ ++ ++++++++ u32 first, nr, i;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!run_written_range(run, &s, &l))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ first = (s - run->start) >> bits;
+++++++++++ ++ ++ ++++++++ nr = l >> bits;
+++++++++++ ++ ++ ++++++++ for (i = first; i < first + nr; i++)
+++++++++++ ++ ++ ++++++++ if (!test_bit(i, run->live))
+++++++++++ ++ ++ ++++++++ not_live++;
+++++++++++ ++ ++ ++++++++ bitmap_set(run->written, first, nr);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ /* Per-stripe log settles wait on the written map, not on a drain. */
+++++++++++ ++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ if (not_live)
+++++++++++ ++ ++ ++++++++ atomic64_add(not_live, &fs_info->stripe_park_stats.written_not_live);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/* Has @run allocated sectors in [start, start + len) it has not yet written? */
+++++++++++ ++ ++ ++++++++static bool run_unwritten_in(struct btrfs_open_stripe_run *run, u64 start,
+++++++++++ ++ ++ ++++++++ u64 len)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ const u32 bits = run->bg->fs_info->sectorsize_bits;
+++++++++++ ++ ++ ++++++++ u32 first, last, i;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!run_written_range(run, &start, &len))
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ first = (start - run->start) >> bits;
+++++++++++ ++ ++ ++++++++ last = first + (len >> bits);
+++++++++++ ++ ++ ++++++++ for (i = first; i < last; i++)
+++++++++++ ++ ++ ++++++++ if (test_bit(i, run->live) && !test_bit(i, run->written))
+++++++++++ ++ ++ ++++++++ return true;
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * As btrfs_stripe_run_incomplete(), returning the class of the run that
+++++++++++ ++ ++ ++++++++ * leaves the stripe incomplete, or -1 if none does.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++int btrfs_stripe_run_incomplete_class(struct btrfs_fs_info *fs_info,
+++++++++++ ++ ++ ++++++++ u64 stripe_start, u64 stripe_len)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ int ret = -1;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++ ++ ++ ++++++++ return -1;
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ return -1;
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if ((run->open && run->offset > stripe_start &&
+++++++++++ ++ ++ ++++++++ run->offset < stripe_start + stripe_len) ||
+++++++++++ ++ ++ ++++++++ run_unwritten_in(run, stripe_start, stripe_len)) {
+++++++++++ ++ ++ ++++++++ ret = run->class;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Is [stripe_start, stripe_start + stripe_len) still waiting for writes from
+++++++++++ ++ ++ ++++++++ * a stripe run? Two ways:
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * - an open run's allocation frontier lies strictly inside it: the run will
+++++++++++ ++ ++ ++++++++ * allocate and write the rest of the stripe later, although every sector
+++++++++++ ++ ++ ++++++++ * allocated so far may already be written;
+++++++++++ ++ ++ ++++++++ * - a run covering it has sectors allocated (live) but not yet written: the
+++++++++++ ++ ++ ++++++++ * frontier has moved past, and the writes are still in flight (writeback
+++++++++++ ++ ++ ++++++++ * allocates a whole range before any of its bios complete).
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Either way the stripe has been written in part, and the rest will arrive
+++++++++++ ++ ++ ++++++++ * as a sub-stripe write that needs the part already written.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++bool btrfs_stripe_run_incomplete(struct btrfs_fs_info *fs_info,
+++++++++++ ++ ++ ++++++++ u64 stripe_start, u64 stripe_len)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ return btrfs_stripe_run_incomplete_class(fs_info, stripe_start,
+++++++++++ ++ ++ ++++++++ stripe_len) >= 0;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/* Open stripe runs by class, for sysfs: @counts has NR_CLASSES entries. */
+++++++++++ ++ ++ ++++++++void btrfs_count_open_stripe_runs(struct btrfs_fs_info *fs_info, int *counts)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ memset(counts, 0, sizeof(*counts) * BTRFS_STRIPE_RUN_NR_CLASSES);
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list) {
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list)
+++++++++++ ++ ++ ++++++++ if (run->open && run->class < BTRFS_STRIPE_RUN_NR_CLASSES)
+++++++++++ ++ ++ ++++++++ counts[run->class]++;
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Like btrfs_stripe_in_open_run(), also reporting the covering run's
+++++++++++ ++ ++ ++++++++ * class: the raid56 parking gate treats NOCOW-class runs specially
+++++++++++ ++ ++ ++++++++ * (sync writes into them never park; see rbio_try_park()).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++bool btrfs_stripe_open_run_class(struct btrfs_fs_info *fs_info, u64 logical,
+++++++++++ ++ ++ ++++++++ enum btrfs_stripe_run_class *class)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ bool ret = false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, logical);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (logical < run->start || logical >= run->end)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Draining runs match too (see btrfs_stripe_in_open_run()):
+++++++++++ ++ ++ ++++++++ * a closed run's end is shrunk to its frontier, so a write
+++++++++++ ++ ++ ++++++++ * into it is below the frontier and its stripes are still
+++++++++++ ++ ++ ++++++++ * exclusive.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (class)
+++++++++++ ++ ++ ++++++++ *class = run->class;
+++++++++++ ++ ++ ++++++++ ret = true;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Report bytes that will never be written: an allocation released before
+++++++++++ ++ ++ ++++++++ * its write was issued. Same accounting as a completion, plus the sectors
+++++++++++ ++ ++ ++++++++ * go dead so a later partial write to their stripe can pad over them
+++++++++++ ++ ++ ++++++++ * instead of falling back to read-modify-write.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_open_stripe_write_abandoned(struct btrfs_block_group *bg, u64 start,
+++++++++++ ++ ++ ++++++++ u64 num_bytes)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (start >= run->start && start < run->offset) {
+++++++++++ ++ ++ ++++++++ ASSERT(start + num_bytes <= run->offset);
+++++++++++ ++ ++ ++++++++ ASSERT(run->inflight_bytes >= num_bytes);
+++++++++++ ++ ++ ++++++++ run_live_set(run, start, num_bytes, false);
+++++++++++ ++ ++ ++++++++ run->inflight_bytes -= num_bytes;
+++++++++++ ++ ++ ++++++++ open_stripe_run_drain_reported(bg, run);
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Report completed IO for an allocation from any stripe run. Every
+++++++++++ ++ ++ ++++++++ * allocated byte must be reported exactly once, here or through
+++++++++++ ++ ++ ++++++++ * btrfs_open_stripe_write_abandoned(), in any number of pieces (e.g. after
+++++++++++ ++ ++ ++++++++ * ordered extent splits).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_open_stripe_write_done(struct btrfs_block_group *bg, u64 start,
+++++++++++ ++ ++ ++++++++ u64 num_bytes)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (start >= run->start && start < run->offset) {
+++++++++++ ++ ++ ++++++++ ASSERT(start + num_bytes <= run->offset);
+++++++++++ ++ ++ ++++++++ ASSERT(run->inflight_bytes >= num_bytes);
+++++++++++ ++ ++ ++++++++ run->inflight_bytes -= num_bytes;
+++++++++++ ++ ++ ++++++++ open_stripe_run_drain_reported(bg, run);
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Range variant of btrfs_open_stripe_write_done() for callers without the
+++++++++++ ++ ++ ++++++++ * block group at hand (e.g. the prealloc path, which never issues data IO
+++++++++++ ++ ++ ++++++++ * for its extent and reports it done at insertion).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_open_stripe_write_done_bytenr(struct btrfs_fs_info *fs_info,
+++++++++++ ++ ++ ++++++++ u64 start, u64 num_bytes)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (list_empty_careful(&fs_info->open_stripe_bgs))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, start);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ btrfs_open_stripe_write_done(bg, start, num_bytes);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Find the open stripe run whose allocated region contains
+++++++++++ ++ ++ ++++++++ * [bytenr, bytenr + num_bytes). The caller must own unreported inflight
+++++++++++ ++ ++ ++++++++ * bytes in that region (i.e. it was allocated from the run and not yet
+++++++++++ ++ ++ ++++++++ * reported done), which is what keeps the returned pointer valid: a run
+++++++++++ ++ ++ ++++++++ * with inflight bytes is never freed.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++struct btrfs_open_stripe_run *btrfs_get_open_stripe_run(
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info, u64 bytenr, u64 num_bytes,
+++++++++++ ++ ++ ++++++++ u64 *open_seq)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run = NULL;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *iter;
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ return NULL;
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(iter, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (bytenr >= iter->start && bytenr < iter->offset) {
+++++++++++ ++ ++ ++++++++ ASSERT(bytenr + num_bytes <= iter->offset);
+++++++++++ ++ ++ ++++++++ ASSERT(iter->inflight_bytes >= num_bytes);
+++++++++++ ++ ++ ++++++++ run = iter;
+++++++++++ ++ ++ ++++++++ *open_seq = iter->open_seq;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ return run;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Pointer-based variant of btrfs_open_stripe_write_done() for callers that
+++++++++++ ++ ++ ++++++++ * recorded the run at allocation or ordered extent creation time. The run
+++++++++++ ++ ++ ++++++++ * pointer stays valid until the reporter's own bytes are reported: a run
+++++++++++ ++ ++ ++++++++ * with inflight bytes is never freed.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * An ordered extent's data IO is over, whether or not all of it reached the
+++++++++++ ++ ++ ++++++++ * disk: stop counting it as in flight, and mark its sectors written. A
+++++++++++ ++ ++ ++++++++ * sector whose write failed or was never issued (truncation, a short direct
+++++++++++ ++ ++ ++++++++ * write, error teardown) will not be written now. Marking it keeps a log
+++++++++++ ++ ++ ++++++++ * settle from waiting for it forever, and a written sector is never padded,
+++++++++++ ++ ++ ++++++++ * so a later partial write to the stripe reads it instead of zeroing it.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Returns the number of allocated sectors that had not been written, and
+++++++++++ ++ ++ ++++++++ * in @first_unwritten the logical address of the lowest of them (U64_MAX if
+++++++++++ ++ ++ ++++++++ * there are none).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++u32 btrfs_open_stripe_write_done_run(struct btrfs_open_stripe_run *run,
+++++++++++ ++ ++ ++++++++ u64 start, u64 num_bytes,
+++++++++++ ++ ++ ++++++++ u64 *first_unwritten)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg = run->bg;
+++++++++++ ++ ++ ++++++++ const u32 bits = bg->fs_info->sectorsize_bits;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ u64 s = start;
+++++++++++ ++ ++ ++++++++ u64 l = num_bytes;
+++++++++++ ++ ++ ++++++++ u32 unwritten = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ *first_unwritten = U64_MAX;
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ ASSERT(run->inflight_bytes >= num_bytes);
+++++++++++ ++ ++ ++++++++ run->inflight_bytes -= num_bytes;
+++++++++++ ++ ++ ++++++++ if (run_written_range(run, &s, &l)) {
+++++++++++ ++ ++ ++++++++ const u32 first = (s - run->start) >> bits;
+++++++++++ ++ ++ ++++++++ const u32 last = first + (l >> bits);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ for (u32 i = first; i < last; i++) {
+++++++++++ ++ ++ ++++++++ if (!test_bit(i, run->live) || test_bit(i, run->written))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (!unwritten)
+++++++++++ ++ ++ ++++++++ *first_unwritten = run->start + ((u64)i << bits);
+++++++++++ ++ ++ ++++++++ unwritten++;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ bitmap_set(run->written, first, l >> bits);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ /* Under the lock: the run and the group may be gone after it. */
+++++++++++ ++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
+++++++++++ ++ ++ ++++++++ open_stripe_run_drain_reported(bg, run);
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ return unwritten;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Close the block group's open runs that predate @seq and return their
+++++++++++ ++ ++ ++++++++ * unallocated tails to the free space cache. Does not wait for inflight
+++++++++++ ++ ++ ++++++++ * IO. Metadata runs are left to btrfs_retire_meta_stripes() and persistent
+++++++++++ ++ ++ ++++++++ * NOCOW runs stay open unless dead, except when @seq is U64_MAX (read-only,
+++++++++++ ++ ++ ++++++++ * removal, unmount), which closes everything.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void close_block_group_stripe_runs(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ u64 seq)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ int class, band;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++restart:
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ u64 tail_start;
+++++++++++ ++ ++ ++++++++ u64 tail_len;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!run->open || run->open_seq >= seq)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * A persistent NOCOW run holds only its inode's
+++++++++++ ++ ++ ++++++++ * write-hole-waived data, so commit retirement protects
+++++++++++ ++ ++ ++++++++ * nothing there, and closing it each commit would waste a
+++++++++++ ++ ++ ++++++++ * stripe per commit on a slowly appended file. It closes at
+++++++++++ ++ ++ ++++++++ * forced quiescing (seq == U64_MAX), at the inode's eviction,
+++++++++++ ++ ++ ++++++++ * or once it is dead.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (run->persistent && seq != U64_MAX) {
+++++++++++ ++ ++ ++++++++ /* A dead run protects nothing, only strands stripes. */
+++++++++++ ++ ++ ++++++++ if (!open_stripe_run_dead(run))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_run_stats.nocow_run_dead_closed);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Metadata runs are still being filled at this point in the
+++++++++++ ++ ++ ++++++++ * commit and their tree blocks are not written until later;
+++++++++++ ++ ++ ++++++++ * btrfs_retire_meta_stripes() closes them once they are.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class) && seq != U64_MAX)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
+++++++++++ ++ ++ ++++++++ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+++++++++++ ++ ++ ++++++++ if (bg->open_stripe[class][band] == run)
+++++++++++ ++ ++ ++++++++ bg->open_stripe[class][band] = NULL;
+++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Closing may have freed the run, and returning its tail to
+++++++++++ ++ ++ ++++++++ * the free space cache needs the lock dropped, so rescan.
+++++++++++ ++ ++ ++++++++ * Each pass closes one run and open runs are bounded (band
+++++++++++ ++ ++ ++++++++ * slots plus the private runs of inodes logged this window).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (tail_len)
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ goto restart;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Close all of a block group's open stripe runs without waiting for their
+++++++++++ ++ ++ ++++++++ * inflight IO to drain.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_close_bg_open_stripes(struct btrfs_block_group *bg)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ close_block_group_stripe_runs(bg, U64_MAX);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/* No run opened before @seq remains (open or draining). */
+++++++++++ ++ ++ ++++++++static bool bg_open_stripes_settled(struct btrfs_block_group *bg, u64 seq)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ bool ret = true;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ /* Persistent nocow runs are not part of any commit window. */
+++++++++++ ++ ++ ++++++++ if (run->persistent && seq != U64_MAX)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /* Nor are retained relocation runs, once nothing is in flight. */
+++++++++++ ++ ++ ++++++++ if (run->retained && open_stripe_run_drained(run) &&
+++++++++++ ++ ++ ++++++++ seq != U64_MAX)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /* Metadata drains after tree writeback, not here. */
+++++++++++ ++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class) && seq != U64_MAX)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (run->open_seq < seq) {
+++++++++++ ++ ++ ++++++++ ret = false;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Stop retaining @bg's relocation runs: free the ones already drained, and
+++++++++++ ++ ++ ++++++++ * let any still open or in flight be freed when they drain, like any other.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void release_retained_stripe_runs(struct btrfs_block_group *bg)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run, *tmp;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry_safe(run, tmp, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (!run->retained)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ run->retained = false;
+++++++++++ ++ ++ ++++++++ atomic64_inc(&bg->fs_info->stripe_run_stats.reloc_run_released);
+++++++++++ ++ ++ ++++++++ if (open_stripe_run_drained(run))
+++++++++++ ++ ++ ++++++++ free_open_stripe_run(bg, run);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Relocation is done with its destination stripes: release its retained
+++++++++++ ++ ++ ++++++++ * runs in every block group (the dedication of btrfs_release_data_reloc_bg()
+++++++++++ ++ ++ ++++++++ * can have moved between groups during the relocation).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void btrfs_release_stripe_reloc_runs(struct btrfs_fs_info *fs_info)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
+++++++++++ ++ ++ ++++++++ release_retained_stripe_runs(bg);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/* Does @bg have metadata stripe runs? */
+++++++++++ ++ ++ ++++++++static bool bg_has_meta_runs(struct btrfs_block_group *bg)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ bool ret = false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class)) {
+++++++++++ ++ ++ ++++++++ ret = true;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Close and drain all of a block group's stripe runs. The caller must
+++++++++++ ++ ++ ++++++++ * prevent new allocations first (e.g. the block group is read-only),
+++++++++++ ++ ++ ++++++++ * otherwise this can wait forever. Membership in fs_info->open_stripe_bgs
+++++++++++ ++ ++ ++++++++ * is left for the next retire walk to collect.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * The wait also ends when the filesystem is in error: after a transaction
+++++++++++ ++ ++ ++++++++ * abort the ordered extents holding a run's bytes may never complete. The
+++++++++++ ++ ++ ++++++++ * runs are then left for btrfs_free_open_stripe_runs() at teardown.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_retire_block_group_stripes(struct btrfs_block_group *bg)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++ ++ ++ ++++++++ bool warned = false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Nothing may outlive this, relocation's retained runs included. */
+++++++++++ ++ ++ ++++++++ release_retained_stripe_runs(bg);
+++++++++++ ++ ++ ++++++++ close_block_group_stripe_runs(bg, U64_MAX);
+++++++++++ ++ ++ ++++++++ /* Parked partial-stripe rbios hold the very bios the wait drains. */
+++++++++++ ++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, bg->start, bg->length, false);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Metadata runs drain when their tree blocks are written and are
+++++++++++ ++ ++ ++++++++ * freed by btrfs_retire_meta_stripes(), both in a commit. Commit now
+++++++++++ ++ ++ ++++++++ * instead of waiting for the next one; no caller holds a transaction
+++++++++++ ++ ++ ++++++++ * handle.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (bg_has_meta_runs(bg))
+++++++++++ ++ ++ ++++++++ btrfs_commit_current_transaction(fs_info->tree_root);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * No timeout: a run that does not drain has an allocation that was
+++++++++++ ++ ++ ++++++++ * never reported written or abandoned, an accounting bug to fix, not
+++++++++++ ++ ++ ++++++++ * to wait out. Say which runs once, and keep waiting.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ while (!wait_var_event_timeout(&bg->open_stripe_runs,
+++++++++++ ++ ++ ++++++++ bg_open_stripes_settled(bg, U64_MAX) ||
+++++++++++ ++ ++ ++++++++ BTRFS_FS_ERROR(fs_info), 60 * HZ)) {
+++++++++++ ++ ++ ++++++++ if (warned)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ btrfs_warn(fs_info,
+++++++++++ ++ ++ ++++++++"block group %llu read-only transition waiting for stripe runs that do not drain",
+++++++++++ ++ ++ ++++++++ bg->start);
+++++++++++ ++ ++ ++++++++ btrfs_dump_open_stripe_runs(bg);
+++++++++++ ++ ++ ++++++++ warned = true;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (BTRFS_FS_ERROR(fs_info) && !bg_open_stripes_settled(bg, U64_MAX))
+++++++++++ ++ ++ ++++++++ btrfs_warn(fs_info,
+++++++++++ ++ ++ ++++++++"block group %llu read-only with stripe runs still in flight after a filesystem error; not waiting for them",
+++++++++++ ++ ++ ++++++++ bg->start);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * The filesystem has gone into error: wake every task waiting for a block
+++++++++++ ++ ++ ++++++++ * group's stripe runs to drain, so the waits that give up on an error
+++++++++++ ++ ++ ++++++++ * (btrfs_retire_block_group_stripes()) notice it. A run's own completion
+++++++++++ ++ ++ ++++++++ * reports may never arrive now.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_wake_open_stripe_waiters(struct btrfs_fs_info *fs_info)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * The caller set fs_error without a lock, and wake_up_var() checks
+++++++++++ ++ ++ ++++++++ * for waiters without one: order the store before that check, or a
+++++++++++ ++ ++ ++++++++ * waiter that has just tested BTRFS_FS_ERROR is missed. Pairs with
+++++++++++ ++ ++ ++++++++ * the barrier in prepare_to_wait_event().
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ smp_mb();
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
+++++++++++ ++ ++ ++++++++ wake_up_var(&bg->open_stripe_runs);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Teardown: free every remaining stripe run and drop each group's
+++++++++++ ++ ++ ++++++++ * membership reference. Needed when no final commit retired them: a mount
+++++++++++ ++ ++ ++++++++ * that fails after log replay allocated tree blocks, or an unmount after a
+++++++++++ ++ ++ ++++++++ * transaction abort. IO is quiesced by now, so a run still counting
+++++++++++ ++ ++ ++++++++ * inflight bytes is reported and freed anyway.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Ordered extents point at their runs (ordered->stripe_run), so this relies
+++++++++++ ++ ++ ++++++++ * on none being left: at unmount every inode, and with it every ordered
+++++++++++ ++ ++ ++++++++ * extent, is gone before close_ctree() runs, delayed iputs are drained, and
+++++++++++ ++ ++ ++++++++ * on both paths the workers that complete ordered extents are stopped first.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_free_open_stripe_runs(struct btrfs_fs_info *fs_info)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ LIST_HEAD(bgs);
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg, *tmp_bg;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ list_splice_init(&fs_info->open_stripe_bgs, &bgs);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ list_for_each_entry_safe(bg, tmp_bg, &bgs, open_stripe_bg_list) {
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run, *tmp;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ list_del_init(&bg->open_stripe_bg_list);
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry_safe(run, tmp, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (run->inflight_bytes)
+++++++++++ ++ ++ ++++++++ btrfs_warn(fs_info,
+++++++++++ ++ ++ ++++++++"stripe run %llu-%llu in block group %llu released at teardown with %llu bytes still in flight",
+++++++++++ ++ ++ ++++++++ run->start, run->end, bg->start,
+++++++++++ ++ ++ ++++++++ run->inflight_bytes);
+++++++++++ ++ ++ ++++++++ list_del(&run->list);
+++++++++++ ++ ++ ++++++++ bitmap_free(run->live);
+++++++++++ ++ ++ ++++++++ bitmap_free(run->written);
+++++++++++ ++ ++ ++++++++ kfree(run);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ memset(bg->open_stripe, 0, sizeof(bg->open_stripe));
+++++++++++ ++ ++ ++++++++ bg->stripe_open_remainder = 0;
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Close and drain the metadata stripe runs once every tree block this
+++++++++++ ++ ++ ++++++++ * transaction allocated has reached the disk.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * btrfs_retire_open_stripes() runs earlier in the commit and skips these
+++++++++++ ++ ++ ++++++++ * runs, so they stay open for the commit's own tree block allocations.
+++++++++++ ++ ++ ++++++++ * Closing earlier would strand blocks the commit has yet to allocate;
+++++++++++ ++ ++ ++++++++ * draining earlier would wait for writes the commit has yet to issue.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_retire_meta_stripes(struct btrfs_fs_info *fs_info, u64 transid)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ LIST_HEAD(retire_list);
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * No option test: a runtime disable leaves the running transaction's
+++++++++++ ++ ++ ++++++++ * metadata runs open, and the commit that follows still has to close
+++++++++++ ++ ++ ++++++++ * and drain them or their block groups never drop their references.
+++++++++++ ++ ++ ++++++++ * The list is empty when nothing is open, so this costs nothing on a
+++++++++++ ++ ++ ++++++++ * filesystem that never had the policy.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
+++++++++++ ++ ++ ++++++++ list_add_tail(&bg->open_stripe_retire_list, &retire_list);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ while (!list_empty(&retire_list)) {
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ bool again = true;
+++++++++++ ++ ++ ++++++++ bool warned = false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ bg = list_first_entry(&retire_list, struct btrfs_block_group,
+++++++++++ ++ ++ ++++++++ open_stripe_retire_list);
+++++++++++ ++ ++ ++++++++ list_del_init(&bg->open_stripe_retire_list);
+++++++++++ ++ ++ ++++++++ if (!(bg->flags & (BTRFS_BLOCK_GROUP_METADATA |
+++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ while (again) {
+++++++++++ ++ ++ ++++++++ u64 tail_start;
+++++++++++ ++ ++ ++++++++ u64 tail_len = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ again = false;
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (!btrfs_stripe_run_class_is_meta(run->class) ||
+++++++++++ ++ ++ ++++++++ !run->open)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * The running transaction's runs are not ours
+++++++++++ ++ ++ ++++++++ * to close: their tree blocks are written, and
+++++++++++ ++ ++ ++++++++ * their stripes settled, by that transaction's
+++++++++++ ++ ++ ++++++++ * own commit.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (run->transid > transid)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ for (int band = 0;
+++++++++++ ++ ++ ++++++++ band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+++++++++++ ++ ++ ++++++++ if (bg->open_stripe[BTRFS_STRIPE_RUN_META][band] == run)
+++++++++++ ++ ++ ++++++++ bg->open_stripe[BTRFS_STRIPE_RUN_META][band] = NULL;
+++++++++++ ++ ++ ++++++++ tail_len = close_open_stripe_run(bg, run, &tail_start);
+++++++++++ ++ ++ ++++++++ again = true;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (tail_len)
+++++++++++ ++ ++ ++++++++ btrfs_add_free_space(bg, tail_start, tail_len);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * The closes above shrank the group's open-run remainder;
+++++++++++ ++ ++ ++++++++ * fold the delta into the space_info now. While the policy
+++++++++++ ++ ++ ++++++++ * is on the next allocation from this group would carry it,
+++++++++++ ++ ++ ++++++++ * but after a runtime disable nothing allocates here again
+++++++++++ ++ ++ ++++++++ * and the stale remainder holds metadata back for good.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ stripe_open_remainder_sync(bg);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Every tree block is on disk, so end_bbio_meta_write() has
+++++++++++ ++ ++ ++++++++ * reported all of them and this normally does not wait. A run
+++++++++++ ++ ++ ++++++++ * that does not drain has a tree block that was freed without
+++++++++++ ++ ++ ++++++++ * reporting to its run, an accounting bug: say which runs
+++++++++++ ++ ++ ++++++++ * once, and keep waiting. Only a filesystem error ends the
+++++++++++ ++ ++ ++++++++ * wait early.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ while (!wait_var_event_timeout(&bg->open_stripe_runs,
+++++++++++ ++ ++ ++++++++ !bg_has_undrained_meta_runs(bg, transid) ||
+++++++++++ ++ ++ ++++++++ BTRFS_FS_ERROR(fs_info), 60 * HZ)) {
+++++++++++ ++ ++ ++++++++ if (warned)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ btrfs_warn(fs_info,
+++++++++++ ++ ++ ++++++++"metadata stripe runs in block group %llu do not drain; a tree block was freed without reporting to its run",
+++++++++++ ++ ++ ++++++++ bg->start);
+++++++++++ ++ ++ ++++++++ btrfs_dump_open_stripe_runs(bg);
+++++++++++ ++ ++ ++++++++ warned = true;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Drained and no longer needed for padding or merging. */
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ while (true) {
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *next = NULL;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (btrfs_stripe_run_class_is_meta(run->class) &&
+++++++++++ ++ ++ ++++++++ open_stripe_run_drained(run)) {
+++++++++++ ++ ++ ++++++++ next = run;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (!next)
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ free_open_stripe_run(bg, next);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Should this tree block be left dirty for the transaction commit to write?
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * A stripe of tree blocks goes out as one full-stripe write only if all of
+++++++++++ ++ ++ ++++++++ * its blocks are written in the same pass. Opportunistic writeback (memory
+++++++++++ ++ ++ ++++++++ * reclaim, the periodic timer) does not know about stripes and would split
+++++++++++ ++ ++ ++++++++ * it into read-modify-writes, so it is declined here, as zoned mode does in
+++++++++++ ++ ++ ++++++++ * btrfs_check_meta_write_pointer(), and a commit is requested instead.
+++++++++++ ++ ++ ++++++++ * WB_SYNC_ALL is never deferred, so a caller that needs the pages written
+++++++++++ ++ ++ ++++++++ * can always get them written.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++bool btrfs_defer_stripe_meta_write(struct btrfs_fs_info *fs_info,
+++++++++++ ++ ++ ++++++++ struct extent_buffer *eb,
+++++++++++ ++ ++ ++++++++ const struct writeback_control *wbc)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ bool defer;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ if (wbc->sync_mode == WB_SYNC_ALL)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, eb->start);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ defer = btrfs_is_stripe_meta_bg(bg);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (defer) {
+++++++++++ ++ ++ ++++++++ set_bit(BTRFS_FS_NEED_TRANS_COMMIT, &fs_info->flags);
+++++++++++ ++ ++ ++++++++ wake_up_process(fs_info->transaction_kthread);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ return defer;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Commit-time retirement: close every data stripe run opened before this
+++++++++++ ++ ++ ++++++++ * call and wait for its IO to complete. Metadata runs are retired later by
+++++++++++ ++ ++ ++++++++ * btrfs_retire_meta_stripes(), and persistent NOCOW runs survive. Must run
+++++++++++ ++ ++ ++++++++ * while the committing transaction still accepts joins (before
+++++++++++ ++ ++ ++++++++ * TRANS_STATE_COMMIT_DOING): tasks holding unreported run bytes can block on
+++++++++++ ++ ++ ++++++++ * a join, and this drain waits for their IO. Records the bumped sequence in
+++++++++++ ++ ++ ++++++++ * @trans; ordered extents from runs with open_seq >= it defer their file
+++++++++++ ++ ++ ++++++++ * extents to the next transaction. Calls are serialized by the transaction
+++++++++++ ++ ++ ++++++++ * commit; @trans is NULL for the final cleanup call.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_retire_open_stripes(struct btrfs_fs_info *fs_info,
+++++++++++ ++ ++ ++++++++ struct btrfs_transaction *trans)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ LIST_HEAD(retire_list);
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ u64 seq;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ seq = ++fs_info->stripe_retire_seq;
+++++++++++ ++ ++ ++++++++ if (trans)
+++++++++++ ++ ++ ++++++++ trans->stripe_retire_seq = seq;
+++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &fs_info->open_stripe_bgs, open_stripe_bg_list)
+++++++++++ ++ ++ ++++++++ list_add_tail(&bg->open_stripe_retire_list, &retire_list);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &retire_list, open_stripe_retire_list)
+++++++++++ ++ ++ ++++++++ close_block_group_stripe_runs(bg, seq);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Parked partial-stripe write rbios (see raid56.c) hold bios whose
+++++++++++ ++ ++ ++++++++ * write_done the drain below waits for; kick them all down first.
+++++++++++ ++ ++ ++++++++ * A racing park that saw its run still open lands after this flush
+++++++++++ ++ ++ ++++++++ * and is bounded by the park timer, not by us.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, 0, U64_MAX, false);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ while (!list_empty(&retire_list)) {
+++++++++++ ++ ++ ++++++++ bg = list_first_entry(&retire_list, struct btrfs_block_group,
+++++++++++ ++ ++ ++++++++ open_stripe_retire_list);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Same escape as btrfs_retire_block_group_stripes(): on an
+++++++++++ ++ ++ ++++++++ * errored filesystem the bytes in flight may never be
+++++++++++ ++ ++ ++++++++ * reported, and this wait sits in the transaction commit and
+++++++++++ ++ ++ ++++++++ * in the teardown call from close_ctree().
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ wait_var_event(&bg->open_stripe_runs,
+++++++++++ ++ ++ ++++++++ bg_open_stripes_settled(bg, seq) ||
+++++++++++ ++ ++ ++++++++ BTRFS_FS_ERROR(fs_info));
+++++++++++ ++ ++ ++++++++ if (BTRFS_FS_ERROR(fs_info) && !bg_open_stripes_settled(bg, seq))
+++++++++++ ++ ++ ++++++++ btrfs_warn(fs_info,
+++++++++++ ++ ++ ++++++++"block group %llu has stripe runs still in flight at retirement after a filesystem error; not waiting for them",
+++++++++++ ++ ++ ++++++++ bg->start);
+++++++++++ ++ ++ ++++++++ list_del_init(&bg->open_stripe_retire_list);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Drop the membership if the group has no runs left. */
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ if (list_empty(&bg->open_stripe_runs) &&
+++++++++++ ++ ++ ++++++++ !list_empty(&bg->open_stripe_bg_list)) {
+++++++++++ ++ ++ ++++++++ list_del_init(&bg->open_stripe_bg_list);
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ } else {
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->open_stripe_lock);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Commit-time recompute of stripe_unusable for every stripe_alloc block group
+++++++++++ ++ ++ ++++++++ * whose free space changed this transaction. Trapped free space only settles
+++++++++++ ++ ++ ++++++++ * at commit -- partially filled stripes are returned by the retire above and
+++++++++++ ++ ++ ++++++++ * deleted extents are unpinned into the free space cache -- so this is where
+++++++++++ ++ ++ ++++++++ * the counter can be made authoritative. Called after btrfs_finish_extent_commit
+++++++++++ ++ ++ ++++++++ * so the free space cache reflects this transaction's frees.
+++++++++++ ++ ++ ++++++++ */
++++++++++++++ +++++++++++static void reloc_reserve_resync(struct btrfs_block_group *bg);
++++++++++++++ +++++++++++
+++++++++++ ++ ++ ++++++++void btrfs_scan_stripe_unusable(struct btrfs_fs_info *fs_info, bool force)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_space_info *sinfo;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ list_for_each_entry(sinfo, &fs_info->space_info, list) {
+++++++++++ ++ ++ ++++++++ u64 total = 0;
+++++++++++ ++ ++ ++++++++ u64 total_claimable = 0;
+++++++++++ ++ ++ ++++++++ u64 total_claimable_reloc = 0;
+++++++++++ ++ ++ ++++++++ u64 total_stranded = 0;
+++++++++++ ++ ++ ++++++++ u64 largest = 0;
+++++++++++ ++ ++ ++++++++ u64 unit = 0;
+++++++++++ ++ ++ ++++++++ u32 nr_rw = 0;
+++++++++++ ++ ++ ++++++++ int raid;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Data groups under stripe_alloc and metadata groups under
+++++++++++ ++ ++ ++++++++ * stripe_meta both place extents in whole stripes only, so
+++++++++++ ++ ++ ++++++++ * both carry trapped free space. Mixed groups are refused.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (!(sinfo->flags & (BTRFS_BLOCK_GROUP_DATA |
+++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_METADATA |
+++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ down_read(&sinfo->groups_sem);
+++++++++++ ++ ++ ++++++++ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!(btrfs_raid_array[raid].bg_flag &
+++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_RAID56_MASK))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &sinfo->block_groups[raid],
+++++++++++ ++ ++ ++++++++ list) {
+++++++++++ ++ ++ ++++++++ bool dirty = test_and_clear_bit(
+++++++++++ ++ ++ ++++++++ BLOCK_GROUP_FLAG_STRIPE_UNUSABLE_DIRTY,
+++++++++++ ++ ++ ++++++++ &bg->runtime_flags);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * @force rescans every armed group regardless of
+++++++++++ ++ ++ ++++++++ * the dirty flag (debug: isolates a missed
+++++++++++ ++ ++ ++++++++ * dirty-marking from a genuine cache difference).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (force || dirty)
+++++++++++ ++ ++ ++++++++ btrfs_block_group_rescan_stripe_unusable(bg);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Sum the authoritative per-group values into the space_info
+++++++++++ ++ ++ ++++++++ * total so it cannot drift from repeated arm/disarm/rescan
+++++++++++ ++ ++ ++++++++ * deltas. Disarmed (read-only) groups are excluded; their
+++++++++++ ++ ++ ++++++++ * free space is already accounted as read-only. Every update
+++++++++++ ++ ++ ++++++++ * of a group's values also updates the total under
+++++++++++ ++ ++ ++++++++ * sinfo->lock, so summing under it gives totals that match
+++++++++++ ++ ++ ++++++++ * the groups: claims, arms and disarms that race with the
+++++++++++ ++ ++ ++++++++ * scan are neither lost nor counted twice.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ spin_lock(&sinfo->lock);
+++++++++++ ++ ++ ++++++++ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!(btrfs_raid_array[raid].bg_flag &
+++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_RAID56_MASK))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &sinfo->block_groups[raid],
+++++++++++ ++ ++ ++++++++ list) {
+++++++++++ ++ ++ ++++++++ if (bg->stripe_unusable_ready) {
+++++++++++ ++ ++ ++++++++ total += bg->stripe_unusable;
+++++++++++ ++ ++ ++++++++ total_stranded += READ_ONCE(bg->stripe_stranded);
+++++++++++ ++ ++ ++++++++ total_claimable += bg->stripe_claimable;
++++++++++++++ +++++++++++ /* a held group's are in bytes_reloc_reserve */
++++++++++++++ +++++++++++ if (bg->start == READ_ONCE(fs_info->data_reloc_bg) &&
++++++++++++++ +++++++++++ !test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
++++++++++++++ +++++++++++ &bg->runtime_flags))
+++++++++++ ++ ++ ++++++++ total_claimable_reloc +=
+++++++++++ ++ ++ ++++++++ bg->stripe_claimable;
+++++++++++ ++ ++ ++++++++ nr_rw++;
+++++++++++ ++ ++ ++++++++ largest = max(largest, bg->length);
++++++++++++++ +++++++++++ if (test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
++++++++++++++ +++++++++++ &bg->runtime_flags)) {
++++++++++++++ +++++++++++ spin_lock(&bg->lock);
++++++++++++++ +++++++++++ reloc_reserve_resync(bg);
++++++++++++++ +++++++++++ spin_unlock(&bg->lock);
++++++++++++++ +++++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ unit = max(unit, bg->full_stripe_len);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ up_read(&sinfo->groups_sem);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_unusable = total;
+++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_claimable = total_claimable;
+++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_claimable_reloc = total_claimable_reloc;
+++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_stranded = total_stranded;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Not if the option went off since the check above: the
+++++++++++ ++ ++ ++++++++ * disable path has cleared the margin and reserve, and no
+++++++++++ ++ ++ ++++++++ * later scan would clear them again.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (!(sinfo->flags & BTRFS_BLOCK_GROUP_DATA) &&
+++++++++++ ++ ++ ++++++++ btrfs_test_opt(fs_info, STRIPE_ALLOC)) {
+++++++++++ ++ ++ ++++++++ u64 reserve = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * stripe_meta. A tree block is reserved in bytes but
+++++++++++ ++ ++ ++++++++ * placed in whole stripes: the run it would join is
+++++++++++ ++ ++ ++++++++ * closed at the transaction boundary, so each group
+++++++++++ ++ ++ ++++++++ * claims at least one fresh stripe per transaction.
+++++++++++ ++ ++ ++++++++ * Pre-pay two transactions' worth of such claims, or
+++++++++++ ++ ++ ++++++++ * the last claimable stripes go to claims made for
+++++++++++ ++ ++ ++++++++ * reservations admitted earlier and an admitted tree
+++++++++++ ++ ++ ++++++++ * block finds none, which aborts the transaction.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_margin = 2ULL * nr_rw * unit;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * COW frees trap stripes (a stripe is claimable only
+++++++++++ ++ ++ ++++++++ * once every block in it is dead), and only relocation
+++++++++++ ++ ++ ++++++++ * packs live blocks back into whole stripes, which
+++++++++++ ++ ++ ++++++++ * needs room elsewhere for the moved group. With more
+++++++++++ ++ ++ ++++++++ * than one group, hold back one group's worth, capped
+++++++++++ ++ ++ ++++++++ * at a quarter of the space_info, from all but
+++++++++++ ++ ++ ++++++++ * relocation. Accounting only: a committing
+++++++++++ ++ ++ ++++++++ * transaction may still allocate here.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (nr_rw > 1 && unit)
+++++++++++ ++ ++ ++++++++ reserve = stripe_round_up(min(largest,
+++++++++++ ++ ++ ++++++++ sinfo->total_bytes / 4),
+++++++++++ ++ ++ ++++++++ unit);
+++++++++++ ++ ++ ++++++++ sinfo->bytes_stripe_reserve = reserve;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * The rescan usually lowers the counter (mid-transaction
+++++++++++ ++ ++ ++++++++ * incremental adds overcount conservatively); admission
+++++++++++ ++ ++ ++++++++ * tickets may be waiting on exactly that headroom.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ btrfs_try_granting_tickets(sinfo);
+++++++++++ ++ ++ ++++++++ spin_unlock(&sinfo->lock);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Does stripe-exclusive allocation force COW for an in-place (nocow or
+++++++++++ ++ ++ ++++++++ * prealloc) write at @bytenr? True for any raid56 data group under
+++++++++++ ++ ++ ++++++++ * stripe_alloc, since an in-place write could tear a stripe holding other
+++++++++++ ++ ++ ++++++++ * files' committed extents. The caller may still allow the write when
+++++++++++ ++ ++ ++++++++ * btrfs_stripe_nocow_writable() proves the stripes belong to the inode.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++bool btrfs_stripe_alloc_forces_cow(struct btrfs_fs_info *fs_info, u64 bytenr)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ bool ret = false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
+++++++++++ ++ ++ ++++++++ if (bg) {
+++++++++++ ++ ++ ++++++++ ret = btrfs_is_stripe_alloc_bg(bg);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Write-hole invariant checker, called for every raid56 write (full-stripe
+++++++++++ ++ ++ ++++++++ * and sub-stripe alike). Under stripe_alloc and stripe_meta a stripe may
+++++++++++ ++ ++ ++++++++ * only be written while an open or draining run covers it; a write outside
+++++++++++ ++ ++ ++++++++ * every run means a retired stripe is being rewritten or an allocation
+++++++++++ ++ ++ ++++++++ * bypassed the policy.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Groups that hosted RELOC or NOCOW runs are skipped: those extents are
+++++++++++ ++ ++ ++++++++ * legitimately overwritten in place after their runs drain.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_stripe_check_write(struct btrfs_fs_info *fs_info,
+++++++++++ ++ ++ ++++++++ u64 full_stripe_start, bool sub_stripe)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_open_stripe_run *run;
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ unsigned long flags;
+++++++++++ ++ ++ ++++++++ bool live = false;
+++++++++++ ++ ++ ++++++++ u64 fsl;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ /* the legacy allocator's writes are still landing after a runtime enable */
+++++++++++ ++ ++ ++++++++ if (test_bit(BTRFS_FS_STRIPE_ALLOC_ENABLING, &fs_info->flags))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, full_stripe_start);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Metadata is checked on the same terms once stripe_meta places it:
+++++++++++ ++ ++ ++++++++ * its runs stay open across the commit's tree building but are closed
+++++++++++ ++ ++ ++++++++ * and drained before the commit ends, so a tree block write outside
+++++++++++ ++ ++ ++++++++ * every live run means a stripe was revisited after the transaction
+++++++++++ ++ ++ ++++++++ * that filled it finished -- the same violation as for data.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if ((!btrfs_is_stripe_alloc_bg(bg) && !btrfs_is_stripe_meta_bg(bg)) ||
+++++++++++ ++ ++ ++++++++ test_bit(BLOCK_GROUP_FLAG_STRIPE_RELOC_USED, &bg->runtime_flags) ||
+++++++++++ ++ ++ ++++++++ test_bit(BLOCK_GROUP_FLAG_STRIPE_NOCOW_USED, &bg->runtime_flags))
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ fsl = bg->full_stripe_len;
+++++++++++ ++ ++ ++++++++ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+++++++++++ ++ ++ ++++++++ if (full_stripe_start < run->end &&
+++++++++++ ++ ++ ++++++++ full_stripe_start + fsl > run->start) {
+++++++++++ ++ ++ ++++++++ live = true;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+++++++++++ ++ ++ ++++++++ WARN_ONCE(!live,
+++++++++++ ++ ++ ++++++++"btrfs: %s write to stripe %llu (block group %llu) outside any live stripe run: write hole window violated",
+++++++++++ ++ ++ ++++++++ sub_stripe ? "sub-stripe" : "full-stripe",
+++++++++++ ++ ++ ++++++++ full_stripe_start, bg->start);
+++++++++++ ++ ++ ++++++++out:
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++#endif
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Drop the dedication of a block group to data relocation. Shared by the
+++++++++++ ++ ++ ++++++++ * zoned allocator and the stripe allocation policy; both dedicate one
+++++++++++ ++ ++ ++++++++ * block group at a time via fs_info->data_reloc_bg.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ bool cleared = false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->relocation_bg_lock);
+++++++++++ ++ ++ ++++++++ if (fs_info->data_reloc_bg == bg->start) {
+++++++++++ ++ ++ ++++++++ fs_info->data_reloc_bg = 0;
+++++++++++ ++ ++ ++++++++ cleared = true;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->relocation_bg_lock);
+++++++++++ ++ ++ ++++++++ if (cleared && bg->space_info) {
+++++++++++ ++ ++ ++++++++ spin_lock(&bg->space_info->lock);
+++++++++++ ++ ++ ++++++++ bg->space_info->bytes_stripe_claimable_reloc = 0;
+++++++++++ ++ ++ ++++++++ spin_unlock(&bg->space_info->lock);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Release the block group currently dedicated to data relocation, closing
+++++++++++ ++ ++ ++++++++ * its open stripe runs so ordinary allocations can use the remaining fully
+++++++++++ ++ ++ ++++++++ * free stripes. Used by the stripe allocation policy when relocation of a
+++++++++++ ++ ++ ++++++++ * block group finishes.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_release_data_reloc_bg(struct btrfs_fs_info *fs_info)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ u64 bytenr;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->relocation_bg_lock);
+++++++++++ ++ ++ ++++++++ bytenr = fs_info->data_reloc_bg;
+++++++++++ ++ ++ ++++++++ fs_info->data_reloc_bg = 0;
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->relocation_bg_lock);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!bytenr)
+++++++++++ ++ ++ ++++++++ goto release;
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, bytenr);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ goto release;
+++++++++++ ++ ++ ++++++++ if (bg->space_info) {
+++++++++++ ++ ++ ++++++++ spin_lock(&bg->space_info->lock);
+++++++++++ ++ ++ ++++++++ bg->space_info->bytes_stripe_claimable_reloc = 0;
+++++++++++ ++ ++ ++++++++ spin_unlock(&bg->space_info->lock);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ btrfs_close_bg_open_stripes(bg);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++release:
+++++++++++ ++ ++ ++++++++ btrfs_release_stripe_reloc_runs(fs_info);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
struct btrfs_caching_control *btrfs_get_caching_control(
struct btrfs_block_group *cache)
{
* btrfs_can_overcommit check here, and we need to pass in
* BTRFS_RESERVE_NO_FLUSH to give ourselves the most amount of
* leeway to allow us to mark this block group as read only.
----------- -- -- -------- if (btrfs_can_overcommit(cache->fs_info, sinfo, num_bytes,
----------- -- -- -------- BTRFS_RESERVE_NO_FLUSH))
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * As for data above: a stripe_meta group's trapped bytes are
+++++++++++ ++ ++ ++++++++ * already in the space_info total, do not re-provide them.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ u64 trapped = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (READ_ONCE(cache->stripe_unusable_ready))
+++++++++++ ++ ++ ++++++++ trapped = min(READ_ONCE(cache->stripe_unusable), num_bytes);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * The stripe_meta reserve is the room a group's live blocks
+++++++++++ ++ ++ ++++++++ * move into when it is relocated, and going read-only is the
+++++++++++ ++ ++ ++++++++ * first step of exactly that: count the reserve as available
+++++++++++ ++ ++ ++++++++ * here, or the guard refuses every relocation at the edge by
+++++++++++ ++ ++ ++++++++ * the width of the reserve.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (btrfs_can_overcommit(cache->fs_info, sinfo, num_bytes - trapped,
+++++++++++ ++ ++ ++++++++ BTRFS_RESERVE_NO_FLUSH, true))
+++++++++++ ++ ++ ++++++++ ret = 0;
++++++++++++++ +++++++++++ /*
++++++++++++++ +++++++++++ * Or the relocation reserve can take this group's live tree
++++++++++++++ +++++++++++ * blocks: a failed tree block allocation outside the reserve
++++++++++++++ +++++++++++ * falls back into it. Only with a reserve and something to
++++++++++++++ +++++++++++ * move: an empty group would pass "0 >= 0" with the reserve
++++++++++++++ +++++++++++ * off. Data gets no such exemption: its test above fails only
++++++++++++++ +++++++++++ * while bytes_may_use depends on this group's free space, and
++++++++++++++ +++++++++++ * data writeback has no reserve to fall back on.
+ + */
++++++++++++++ +++++++++++ else if (sinfo->bytes_reloc_reserve &&
++++++++++++++ +++++++++++ cache->used + cache->reserved &&
++++++++++++++ +++++++++++ sinfo->bytes_reloc_reserve >=
++++++++++++++ +++++++++++ cache->used + cache->reserved)
+ + ret = 0;
}
if (!ret) {
list_add_tail(&cache->ro_list, &sinfo->ro_bgs);
}
out:
++++++++++++++ +++++++++++ if (ret)
++++++++++++++ +++++++++++ btrfs_reloc_reserve_rejoin(cache);
spin_unlock(&cache->lock);
spin_unlock(&sinfo->lock);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * A read-only group's free space (trapped included) is now accounted
+++++++++++ ++ ++ ++++++++ * as read-only and excluded from statfs; drop its stripe_unusable
+++++++++++ ++ ++ ++++++++ * contribution so it is not subtracted twice. Idempotent, so repeated
+++++++++++ ++ ++ ++++++++ * ro increments are harmless.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (!ret) {
+++++++++++ ++ ++ ++++++++ btrfs_block_group_disarm_stripe_unusable(cache);
+++++++++++ ++ ++ ++++++++ btrfs_stripe_unmanaged_sync(cache);
+++++++++++ ++ ++ ++++++++ }
if (ret == -ENOSPC && btrfs_test_opt(cache->fs_info, ENOSPC_DEBUG)) {
btrfs_info(cache->fs_info,
"unable to make block group %llu ro", cache->start);
return true;
}
- * row failed with -ENOSPC. Relocating a group needs room for
++++++++++ ++ ++ ++++++++/*
++++++++++ ++ ++ ++++++++ * Reclaim gives up on a space_info for a while after this many reclaims in a
+++++++++++ ++ ++ ++++++++ * row failed with -ENOSPC. Relocating or compacting a group needs room for
++++++++++ ++ ++ ++++++++ * its live data (and metadata reservations for moving it); at the fill edge
++++++++++ ++ ++ ++++++++ * there is none, every attempt fails after doing its work, and retrying each
++++++++++ ++ ++ ++++++++ * failed group on every worker run held the balance exclusive operation for
++++++++++ ++ ++ ++++++++ * over an hour while writers starved (octopus P1, bhive 2026-10-20).
++++++++++ ++ ++ ++++++++ */
++++++++++ ++ ++ ++++++++#define BTRFS_RECLAIM_ENOSPC_STREAK 3
++++++++++ ++ ++ ++++++++#define BTRFS_RECLAIM_BACKOFF (10 * 60 * HZ)
++++++++++ ++ ++ ++++++++
++++++++++ ++ ++ ++++++++/* Free space that a retried reclaim could use: free in groups, unallocated. */
++++++++++ ++ ++ ++++++++static u64 reclaim_free_bytes(const struct btrfs_space_info *space_info)
++++++++++ ++ ++ ++++++++{
++++++++++ ++ ++ ++++++++ return space_info->total_bytes - min(space_info->total_bytes,
++++++++++ ++ ++ ++++++++ space_info->bytes_used) +
++++++++++ ++ ++ ++++++++ atomic64_read(&space_info->fs_info->free_chunk_space);
++++++++++ ++ ++ ++++++++}
++++++++++ ++ ++ ++++++++
++++++++++ ++ ++ ++++++++/*
++++++++++ ++ ++ ++++++++ * Is reclaim backing off for this space_info? The backoff ends when its
++++++++++ ++ ++ ++++++++ * time is up, when a chunk was added or removed, or when free space grew by
++++++++++ ++ ++ ++++++++ * what the last failed group needed.
++++++++++ ++ ++ ++++++++ */
++++++++++ ++ ++ ++++++++bool btrfs_reclaim_backoff_active(struct btrfs_space_info *space_info)
++++++++++ ++ ++ ++++++++{
++++++++++ ++ ++ ++++++++ bool active = false;
++++++++++ ++ ++ ++++++++
++++++++++ ++ ++ ++++++++ spin_lock(&space_info->lock);
++++++++++ ++ ++ ++++++++ if (space_info->reclaim_backoff_until) {
++++++++++ ++ ++ ++++++++ active = time_before(jiffies, space_info->reclaim_backoff_until) &&
++++++++++ ++ ++ ++++++++ space_info->total_bytes == space_info->reclaim_backoff_total &&
++++++++++ ++ ++ ++++++++ reclaim_free_bytes(space_info) < space_info->reclaim_backoff_free;
++++++++++ ++ ++ ++++++++ if (!active) {
++++++++++ ++ ++ ++++++++ space_info->reclaim_backoff_until = 0;
++++++++++ ++ ++ ++++++++ space_info->reclaim_enospc_streak = 0;
++++++++++ ++ ++ ++++++++ }
++++++++++ ++ ++ ++++++++ }
++++++++++ ++ ++ ++++++++ spin_unlock(&space_info->lock);
++++++++++ ++ ++ ++++++++ return active;
++++++++++ ++ ++ ++++++++}
++++++++++ ++ ++ ++++++++
++++++++++ ++ ++ ++++++++/*
++++++++++ ++ ++ ++++++++ * Count a reclaim of @bg that failed with -ENOSPC (@enospc) or succeeded,
++++++++++ ++ ++ ++++++++ * and start the backoff at the end of a streak of failures. @need is what
++++++++++ ++ ++ ++++++++ * the group held: free space growing by that much ends the backoff early.
++++++++++ ++ ++ ++++++++ */
++++++++++ ++ ++ ++++++++static void reclaim_note_result(struct btrfs_block_group *bg, bool enospc, u64 need)
++++++++++ ++ ++ ++++++++{
++++++++++ ++ ++ ++++++++ struct btrfs_space_info *space_info = bg->space_info;
++++++++++ ++ ++ ++++++++ u32 streak;
++++++++++ ++ ++ ++++++++
++++++++++ ++ ++ ++++++++ spin_lock(&space_info->lock);
++++++++++ ++ ++ ++++++++ if (!enospc) {
++++++++++ ++ ++ ++++++++ space_info->reclaim_enospc_streak = 0;
++++++++++ ++ ++ ++++++++ spin_unlock(&space_info->lock);
++++++++++ ++ ++ ++++++++ return;
++++++++++ ++ ++ ++++++++ }
++++++++++ ++ ++ ++++++++ streak = ++space_info->reclaim_enospc_streak;
++++++++++ ++ ++ ++++++++ if (streak >= BTRFS_RECLAIM_ENOSPC_STREAK) {
++++++++++ ++ ++ ++++++++ space_info->reclaim_backoff_until = (jiffies + BTRFS_RECLAIM_BACKOFF) ?: 1;
++++++++++ ++ ++ ++++++++ space_info->reclaim_backoff_total = space_info->total_bytes;
++++++++++ ++ ++ ++++++++ space_info->reclaim_backoff_free = reclaim_free_bytes(space_info) +
++++++++++ ++ ++ ++++++++ max_t(u64, need, SZ_16M);
++++++++++ ++ ++ ++++++++ space_info->reclaim_backoffs++;
++++++++++ ++ ++ ++++++++ }
++++++++++ ++ ++ ++++++++ spin_unlock(&space_info->lock);
++++++++++ ++ ++ ++++++++ if (streak >= BTRFS_RECLAIM_ENOSPC_STREAK)
++++++++++ ++ ++ ++++++++ btrfs_debug(bg->fs_info,
++++++++++ ++ ++ ++++++++"reclaim: %u reclaims failed with ENOSPC (last block group %llu), backing off flags 0x%llx",
++++++++++ ++ ++ ++++++++ streak, bg->start, space_info->flags);
++++++++++ ++ ++ ++++++++}
++++++++++ ++ ++ ++++++++
+++++++++++++++++ ++++++++/* A refused operation's retry window is open: see btrfs_exclop_start(). */
+++++++++++++++++ ++++++++static bool reclaim_yielding(const struct btrfs_fs_info *fs_info)
+++++++++++++++++ ++++++++{
+++++++++++++++++ ++++++++ const unsigned long until = READ_ONCE(fs_info->reclaim_yield_until);
+++++++++++++++++ ++++++++
+++++++++++++++++ ++++++++ return until && time_before(jiffies, until);
+++++++++++++++++ ++++++++}
+++++++++++++++++ ++++++++
+++++++++++++++++ ++++++++/*
+++++++++++++++++ ++++++++ * Unmounting, or read-only after an error: every group taken from here on is
+++++++++++++++++ ++++++++ * dropped (see btrfs_need_cleaner_sleep()), so do not wait out a gap for them.
+++++++++++++++++ ++++++++ */
+++++++++++++++++ ++++++++static bool reclaim_stopping(const struct btrfs_fs_info *fs_info)
+++++++++++++++++ ++++++++{
+++++++++++++++++ ++++++++ return btrfs_need_cleaner_sleep(fs_info) || BTRFS_FS_ERROR(fs_info);
+++++++++++++++++ ++++++++}
+++++++++++++++++ ++++++++
void btrfs_reclaim_bgs_work(struct work_struct *work)
{
struct btrfs_fs_info *fs_info =
if (btrfs_fs_closing(fs_info))
return;
- /* Relocation cannot proceed on an errored filesystem. */
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Relocation cannot proceed, and the stripe drain it starts with
+++++++++++ ++ ++ ++++++++ * would wait for IO that will not complete.
+++++++++++ ++ ++ ++++++++ */
++++++++++ ++ ++ ++++++++ if (BTRFS_FS_ERROR(fs_info))
++++++++++ ++ ++ ++++++++ return;
++++++++++ ++ ++ ++++++++
if (!btrfs_should_reclaim(fs_info))
return;
u64 used;
u64 reserved;
u64 old_total;
+++++++++++ ++ ++ ++++++++ u64 trapped;
++++++++++ ++ ++ ++++++++ u64 need;
int ret = 0;
+++++++++++ ++ ++ ++++++++ u8 forced;
++++++++++ ++ ++ ++++++++ bool tried = false;
+++++++++++++++++ ++++++++ bool handoff = false;
+++++++++++ ++ ++ ++++++++ bool compact = false;
+++++++++++ ++ ++ ++++++++ bool compacted = false;
bg = list_first_entry(&fs_info->reclaim_bgs,
struct btrfs_block_group,
space_info = bg->space_info;
spin_unlock(&fs_info->unused_bgs_lock);
- * succeed.
++++++++++ ++ ++ ++++++++ need = READ_ONCE(bg->used);
++++++++++ ++ ++ ++++++++ /*
++++++++++ ++ ++ ++++++++ * Backing off: drop the group rather than requeue it. The
++++++++++ ++ ++ ++++++++ * rescans and usage updates queue it again once reclaim may
- if (btrfs_reclaim_backoff_active(space_info))
+++++++++++ ++ ++ ++++++++ * succeed. A stripe_reclaim_now request is tried regardless.
++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (!READ_ONCE(bg->stripe_reclaim_forced) &&
+++++++++++ ++ ++ ++++++++ btrfs_reclaim_backoff_active(space_info))
++++++++++ ++ ++ ++++++++ goto next;
++++++++++ ++ ++ ++++++++ tried = true;
++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * A stripe_alloc / stripe_meta group's live extents can only
+++++++++++ ++ ++ ++++++++ * move into whole free stripes, and relocation never waits for
+++++++++++ ++ ++ ++++++++ * them. Once the worker has emptied and removed enough groups,
+++++++++++ ++ ++ ++++++++ * the ones left are the trapped ones it was queued to move next
+++++++++++ ++ ++ ++++++++ * with next to nothing claimable, while gigabytes may sit
+++++++++++ ++ ++ ++++++++ * unallocated: give it a chunk to relocate into, once per
+++++++++++ ++ ++ ++++++++ * attempt. Here rather than in the reservation path so that a
+++++++++++ ++ ++ ++++++++ * device shrink, which relocates without room by design, keeps
+++++++++++ ++ ++ ++++++++ * failing cleanly instead of growing into space it cannot use.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (btrfs_is_stripe_alloc_bg(bg) || btrfs_is_stripe_meta_bg(bg)) {
+++++++++++ ++ ++ ++++++++ u64 elsewhere = READ_ONCE(space_info->bytes_stripe_claimable);
+++++++++++ ++ ++ ++++++++ u64 need = READ_ONCE(bg->used);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ forced = READ_ONCE(bg->stripe_reclaim_forced);
+++++++++++ ++ ++ ++++++++ /* Compaction moves at most half of it. */
+++++++++++ ++ ++ ++++++++ if (forced ? forced == STRIPE_RECLAIM_COMPACT :
+++++++++++ ++ ++ ++++++++ btrfs_stripe_bg_wants_compaction(bg))
+++++++++++ ++ ++ ++++++++ need /= 2;
+++++++++++ ++ ++ ++++++++ elsewhere -= min(elsewhere, READ_ONCE(bg->stripe_claimable));
+++++++++++ ++ ++ ++++++++ if (elsewhere < need) {
+++++++++++ ++ ++ ++++++++ struct btrfs_trans_handle *trans;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ trans = btrfs_join_transaction(fs_info->tree_root);
+++++++++++ ++ ++ ++++++++ if (!IS_ERR(trans)) {
+++++++++++ ++ ++ ++++++++ btrfs_chunk_alloc(trans, space_info,
+++++++++++ ++ ++ ++++++++ btrfs_get_alloc_profile(fs_info,
+++++++++++ ++ ++ ++++++++ space_info->flags),
+++++++++++ ++ ++ ++++++++ CHUNK_ALLOC_FORCE);
+++++++++++ ++ ++ ++++++++ btrfs_end_transaction(trans);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
/* Don't race with allocators so take the groups_sem */
down_write(&space_info->groups_sem);
spin_unlock(&bg->lock);
trace_btrfs_reclaim_block_group(bg);
---------- -- -- -------- ret = btrfs_relocate_chunk(fs_info, bg->start, false);
++++++++++ ++ ++ ++++++++ btrfs_debug(fs_info,
- "reclaim: relocating block group %llu flags 0x%llx length %llu used %llu reserved %llu",
- bg->start, bg->flags, bg->length, used, reserved);
- ret = btrfs_relocate_chunk(fs_info, bg->start, false);
+++++++++++ ++ ++ ++++++++"reclaim: %s block group %llu flags 0x%llx length %llu used %llu reserved %llu trapped %llu",
+++++++++++ ++ ++ ++++++++ compact ? "compacting" : "relocating", bg->start,
+++++++++++ ++ ++ ++++++++ bg->flags, bg->length, used, reserved, trapped);
+++++++++++ ++ ++ ++++++++ if (compact) {
+++++++++++ ++ ++ ++++++++ ret = btrfs_compact_chunk(fs_info, bg->start, &compacted);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Compacted, the group stays: drop our read-only hold.
+++++++++++ ++ ++ ++++++++ * It gave back trapped space, not a chunk; what moved
+++++++++++ ++ ++ ++++++++ * is counted in stripe_run_stats.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (!ret && compacted) {
+++++++++++ ++ ++ ++++++++ btrfs_dec_block_group_ro(bg);
+++++++++++ ++ ++ ++++++++ used = 0;
+++++++++++ ++ ++ ++++++++ reserved = 0;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ } else {
+++++++++++ ++ ++ ++++++++ ret = btrfs_relocate_chunk(fs_info, bg->start, false);
+++++++++++ ++ ++ ++++++++ }
if (ret) {
btrfs_dec_block_group_ro(bg);
-------------- ----------- btrfs_err(fs_info, "error relocating chunk %llu",
-------------- ----------- bg->start);
++++++++++++++ +++++++++++ btrfs_err(fs_info, "error relocating chunk %llu: %d",
++++++++++++++ +++++++++++ bg->start, ret);
used = 0;
reserved = 0;
spin_lock(&space_info->lock);
if (space_info->total_bytes < old_total)
btrfs_set_periodic_reclaim_ready(space_info, true);
spin_unlock(&space_info->lock);
- btrfs_debug(fs_info, "reclaim: relocated block group %llu",
++++++++++ ++ ++ ++++++++ if (!ret) {
+++++++++++ ++ ++ ++++++++ btrfs_debug(fs_info, "reclaim: %s block group %llu",
+++++++++++ ++ ++ ++++++++ compacted ? "compacted" : "relocated",
++++++++++ ++ ++ ++++++++ bg->start);
++++++++++ ++ ++ ++++++++ reclaim_note_result(bg, false, need);
++++++++++ ++ ++ ++++++++ }
next:
---------- -- -- -------- if (ret && !READ_ONCE(space_info->periodic_reclaim))
++++++++++ ++ ++ ++++++++ if (tried && ret == -ENOSPC)
++++++++++ ++ ++ ++++++++ reclaim_note_result(bg, true, need);
++++++++++ ++ ++ ++++++++ /* An ENOSPC failure is not retried at once; see above. */
++++++++++ ++ ++ ++++++++ if (ret && ret != -ENOSPC &&
++++++++++ ++ ++ ++++++++ !READ_ONCE(space_info->periodic_reclaim))
btrfs_link_bg_list(bg, &retry_list);
btrfs_put_block_group(bg);
spin_unlock(&fs_info->unused_bgs_lock);
}
----------- -- -- --------void btrfs_mark_bg_to_reclaim(struct btrfs_block_group *bg)
+++++++++++ ++ ++ ++++++++/* Returns true when the group was linked to the reclaim list. */
+++++++++++ ++ ++ ++++++++bool btrfs_mark_bg_to_reclaim(struct btrfs_block_group *bg)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = bg->fs_info;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!btrfs_link_bg_list(bg, &fs_info->reclaim_bgs))
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ trace_btrfs_add_reclaim_block_group(bg);
+++++++++++ ++ ++ ++++++++ return true;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Room for a forced reclaim beyond the other groups' claimable stripes: the
+++++++++++ ++ ++ ++++++++ * one chunk the reclaim worker allocates when they fall short (see
+++++++++++ ++ ++ ++++++++ * btrfs_reclaim_block_group()), estimated as the data share of the
+++++++++++ ++ ++ ++++++++ * unallocated device space at @bg's width, capped at @bg's length.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static u64 stripe_reclaim_chunk_room(struct btrfs_block_group *bg)
{
struct btrfs_fs_info *fs_info = bg->fs_info;
- if (btrfs_link_bg_list(bg, &fs_info->reclaim_bgs))
- trace_btrfs_add_reclaim_block_group(bg);
+++++++++++ ++ ++ ++++++++ struct btrfs_chunk_map *map;
+++++++++++ ++ ++ ++++++++ u64 unalloc = atomic64_read(&fs_info->free_chunk_space);
+++++++++++ ++ ++ ++++++++ u64 room;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ map = btrfs_get_chunk_map(fs_info, bg->start, 1);
+++++++++++ ++ ++ ++++++++ if (IS_ERR(map))
+++++++++++ ++ ++ ++++++++ return 0;
+++++++++++ ++ ++ ++++++++ room = div_u64(unalloc * nr_data_stripes(map), map->num_stripes);
+++++++++++ ++ ++ ++++++++ btrfs_free_chunk_map(map);
+++++++++++ ++ ++ ++++++++ return min(room, bg->length);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Queue @bg for the reclaim worker with the reclaim it is to do. Returns
+++++++++++ ++ ++ ++++++++ * false when the group is already on a list (queued, unused or new).
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static bool stripe_reclaim_now_queue(struct btrfs_block_group *bg,
+++++++++++ ++ ++ ++++++++ enum stripe_reclaim_kind kind)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ bool queued;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock(&bg->lock);
+++++++++++ ++ ++ ++++++++ bg->stripe_reclaim_forced = kind;
+++++++++++ ++ ++ ++++++++ queued = btrfs_mark_bg_to_reclaim(bg);
+++++++++++ ++ ++ ++++++++ if (queued)
+++++++++++ ++ ++ ++++++++ bg->stripe_reclaim_jiffies = jiffies ?: 1;
+++++++++++ ++ ++ ++++++++ else
+++++++++++ ++ ++ ++++++++ bg->stripe_reclaim_forced = STRIPE_RECLAIM_NONE;
+++++++++++ ++ ++ ++++++++ spin_unlock(&bg->lock);
+++++++++++ ++ ++ ++++++++ if (queued)
+++++++++++ ++ ++ ++++++++ btrfs_debug(bg->fs_info,
+++++++++++ ++ ++ ++++++++ "reclaim: queued block group %llu to %s (stripe_reclaim_now)",
+++++++++++ ++ ++ ++++++++ bg->start, kind == STRIPE_RECLAIM_COMPACT ?
+++++++++++ ++ ++ ++++++++ "compact" : "relocate");
+++++++++++ ++ ++ ++++++++ return queued;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * /sys/fs/btrfs/<uuid>/stripe_reclaim_now: reclaim stripe_alloc / stripe_meta
+++++++++++ ++ ++ ++++++++ * raid56 block groups now, e.g. from an idle-time job, rather than waiting
+++++++++++ ++ ++ ++++++++ * for the commit-time rescan and its five-minute hysteresis. The threshold
+++++++++++ ++ ++ ++++++++ * is the space_info's bg_reclaim_threshold, or BTRFS_DEFAULT_RECLAIM_THRESH
+++++++++++ ++ ++ ++++++++ * where that is 0 (reclaim off), since the request is explicit.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * @all: queue every such group the reclaim predicate picks at that
+++++++++++ ++ ++ ++++++++ * threshold (relocate below it, compact above it once stripe_compact percent
+++++++++++ ++ ++ ++++++++ * is trapped), ignoring the hysteresis.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Otherwise @start names one group (its logical start): it is reclaimed
+++++++++++ ++ ++ ++++++++ * whatever the trapped minimums say, compacted if it is above the threshold
+++++++++++ ++ ++ ++++++++ * and has trapped stripes, relocated otherwise. -ENOENT: no group starts
+++++++++++ ++ ++ ++++++++ * there; -EINVAL: not a stripe_alloc / stripe_meta raid56 group (relocate
+++++++++++ ++ ++ ++++++++ * any other group with balance's vrange filter); -EBUSY: the group is
+++++++++++ ++ ++ ++++++++ * read-only (balance or scrub hold it); -EAGAIN: it is already on a list
+++++++++++ ++ ++ ++++++++ * (queued, unused or new); -ENOSPC: the other groups' claimable stripes plus
+++++++++++ ++ ++ ++++++++ * one new chunk cannot take what would move.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Both return once the groups are queued, without waiting for the reclaim.
+++++++++++ ++ ++ ++++++++ * The worker takes the balance exclusive operation and gives the list back
+++++++++++ ++ ++ ++++++++ * when another operation holds it, so a request made while a balance,
+++++++++++ ++ ++ ++++++++ * replace, resize or device add/remove runs fails at once with -EAGAIN
+++++++++++ ++ ++ ++++++++ * rather than wait unseen; nothing is queued. The reclaim worker's own hold
+++++++++++ ++ ++ ++++++++ * (a balance with no balance control) is not refused: it picks up the new
+++++++++++ ++ ++ ++++++++ * entries.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++int btrfs_stripe_reclaim_now(struct btrfs_fs_info *fs_info, bool all, u64 start)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_space_info *sinfo;
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ enum btrfs_exclusive_operation op;
+++++++++++ ++ ++ ++++++++ enum stripe_reclaim_kind kind;
+++++++++++ ++ ++ ++++++++ int queued = 0;
+++++++++++ ++ ++ ++++++++ int ret = 0;
+++++++++++ + ++ ++++++++
----------- - -- -------- if (btrfs_link_bg_list(bg, &fs_info->reclaim_bgs))
----------- - -- -------- trace_btrfs_add_reclaim_block_group(bg);
+++++++++++ ++ ++ ++++++++ if (sb_rdonly(fs_info->sb) || BTRFS_FS_ERROR(fs_info))
+++++++++++ ++ ++ ++++++++ return -EROFS;
+++++++++++ ++ ++ ++++++++ spin_lock(&fs_info->super_lock);
+++++++++++ ++ ++ ++++++++ op = fs_info->exclusive_operation;
+++++++++++ ++ ++ ++++++++ spin_unlock(&fs_info->super_lock);
+++++++++++ ++ ++ ++++++++ if (op != BTRFS_EXCLOP_NONE &&
+++++++++++ ++ ++ ++++++++ (op != BTRFS_EXCLOP_BALANCE || READ_ONCE(fs_info->balance_ctl)))
+++++++++++ ++ ++ ++++++++ return -EAGAIN;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!all) {
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, start);
+++++++++++ ++ ++ ++++++++ if (!bg)
+++++++++++ ++ ++ ++++++++ return -ENOENT;
+++++++++++ ++ ++ ++++++++ if (bg->start != start) {
+++++++++++ ++ ++ ++++++++ ret = -ENOENT;
+++++++++++ ++ ++ ++++++++ } else if (!btrfs_is_stripe_alloc_bg(bg) &&
+++++++++++ ++ ++ ++++++++ !btrfs_is_stripe_meta_bg(bg)) {
+++++++++++ ++ ++ ++++++++ ret = -EINVAL;
+++++++++++ ++ ++ ++++++++ } else if (READ_ONCE(bg->ro)) {
+++++++++++ ++ ++ ++++++++ ret = -EBUSY;
+++++++++++ ++ ++ ++++++++ } else {
+++++++++++ ++ ++ ++++++++ int thresh = READ_ONCE(bg->space_info->bg_reclaim_threshold);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ kind = btrfs_stripe_bg_reclaim_now_kind(bg,
+++++++++++ ++ ++ ++++++++ thresh > 0 ? thresh : BTRFS_DEFAULT_RECLAIM_THRESH,
+++++++++++ ++ ++ ++++++++ true, stripe_reclaim_chunk_room(bg));
+++++++++++ ++ ++ ++++++++ if (kind == STRIPE_RECLAIM_NONE)
+++++++++++ ++ ++ ++++++++ ret = READ_ONCE(bg->ro) ? -EBUSY : -ENOSPC;
+++++++++++ ++ ++ ++++++++ else if (stripe_reclaim_now_queue(bg, kind))
+++++++++++ ++ ++ ++++++++ queued = 1;
+++++++++++ ++ ++ ++++++++ else
+++++++++++ ++ ++ ++++++++ ret = -EAGAIN;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ if (ret)
+++++++++++ ++ ++ ++++++++ return ret;
+++++++++++ ++ ++ ++++++++ goto kick;
+++++++++++ ++ ++ ++++++++ }
+
+++++++++++ ++ ++ ++++++++ list_for_each_entry(sinfo, &fs_info->space_info, list) {
+++++++++++ ++ ++ ++++++++ int thresh = READ_ONCE(sinfo->bg_reclaim_threshold);
+++++++++++ ++ ++ ++++++++ int raid;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (thresh <= 0)
+++++++++++ ++ ++ ++++++++ thresh = BTRFS_DEFAULT_RECLAIM_THRESH;
+++++++++++ ++ ++ ++++++++ down_read(&sinfo->groups_sem);
+++++++++++ ++ ++ ++++++++ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
+++++++++++ ++ ++ ++++++++ if (!(btrfs_raid_array[raid].bg_flag &
+++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_RAID56_MASK))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ list_for_each_entry(bg, &sinfo->block_groups[raid], list) {
+++++++++++ ++ ++ ++++++++ if (!btrfs_is_stripe_alloc_bg(bg) &&
+++++++++++ ++ ++ ++++++++ !btrfs_is_stripe_meta_bg(bg))
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ kind = btrfs_stripe_bg_reclaim_now_kind(bg, thresh,
+++++++++++ ++ ++ ++++++++ false, stripe_reclaim_chunk_room(bg));
+++++++++++ ++ ++ ++++++++ if (kind != STRIPE_RECLAIM_NONE &&
+++++++++++ ++ ++ ++++++++ stripe_reclaim_now_queue(bg, kind))
+++++++++++ ++ ++ ++++++++ queued++;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ up_read(&sinfo->groups_sem);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ btrfs_debug(fs_info, "reclaim: stripe_reclaim_now queued %d block groups",
+++++++++++ ++ ++ ++++++++ queued);
+++++++++++ ++ ++ ++++++++kick:
+++++++++++ ++ ++ ++++++++ if (queued)
+++++++++++ ++ ++ ++++++++ queue_work(system_dfl_wq, &fs_info->reclaim_bgs_work);
+++++++++++ ++ ++ ++++++++ return 0;
}
static int read_bg_from_eb(struct btrfs_fs_info *fs_info, const struct btrfs_key *key,
mutex_unlock(&fs_info->ro_block_group_mutex);
btrfs_end_transaction(trans);
- if (ret)
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * The group is read-only: close and drain its open stripe runs. An
+++++++++++ ++ ++ ++++++++ * allocation racing with the read-only transition either installed
+++++++++++ ++ ++ ++++++++ * its run before this (closed and drained here), or observes ->ro
+++++++++++ ++ ++ ++++++++ * under the stripe_run_lock and backs out.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (!ret)
+++++++++++ ++ ++ ++++++++ btrfs_retire_block_group_stripes(cache);
++++++++++++++++++++++++++ else
++++++++++++ +++++++++++++ btrfs_bg_unblock_nocow(cache);
return ret;
}
cache->zone_unusable - cache->used;
sinfo->bytes_readonly -= num_bytes;
list_del_init(&cache->ro_list);
+++++++++++ ++ ++ ++++++++ became_rw = true;
++++++++++++++ +++++++++++ btrfs_reloc_reserve_rejoin(cache);
}
spin_unlock(&cache->lock);
spin_unlock(&sinfo->lock);
spin_unlock(&table->cache_lock);
}
+++++++++++++++++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
+++++++++++++++++ ++++++++/*
+++++++++++++++++ ++++++++ * Test knob (debug/rbio_cache_size): with a small or zero limit, a
+++++++++++++++++ ++++++++ * read-modify-write cannot take its stripe from the cache and must read it,
+++++++++++++++++ ++++++++ * which is what a fault-injecting test needs to see. Lowering it empties the
+++++++++++++++++ ++++++++ * cache, so the new limit holds from the next rbio on.
+++++++++++++++++ ++++++++ */
+++++++++++++++++ ++++++++void btrfs_set_rbio_cache_size(struct btrfs_fs_info *info, unsigned int size)
+++++++++++++++++ ++++++++{
+++++++++++++++++ ++++++++ unsigned int old = READ_ONCE(info->rbio_cache_size);
+++++++++++++++++ ++++++++
+++++++++++++++++ ++++++++ WRITE_ONCE(info->rbio_cache_size, size);
+++++++++++++++++ ++++++++ if (size < old && info->stripe_hash_table)
+++++++++++++++++ ++++++++ btrfs_clear_rbio_cache(info);
+++++++++++++++++ ++++++++}
+++++++++++++++++ ++++++++#endif
+++++++++++++++++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Send every parked rbio down and stop the park timer, before the raid56
+++++++++++ ++ ++ ++++++++ * workqueue is destroyed: the timer queues its expired rbios there. By
+++++++++++ ++ ++ ++++++++ * now nothing should be parked; a few passes cover an rbio that parks
+++++++++++ ++ ++ ++++++++ * again on its way down.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++void btrfs_stop_parked_rbios(struct btrfs_fs_info *fs_info)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_stripe_hash_table *table = fs_info->stripe_hash_table;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!table || !fs_info->rmw_workers)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ for (int i = 0; i < 3 && !list_empty_careful(&table->parked); i++) {
+++++++++++ ++ ++ ++++++++ btrfs_flush_parked_rbios(fs_info, 0, U64_MAX, true);
+++++++++++ ++ ++ ++++++++ flush_workqueue(fs_info->rmw_workers);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ cancel_delayed_work_sync(&table->parked_work);
+++++++++++ ++ ++ ++++++++ flush_workqueue(fs_info->rmw_workers);
+++++++++++ ++ ++ ++++++++ WARN_ON(!list_empty_careful(&table->parked));
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
/*
* remove all cached entries and free the hash table
* used by unmount
static void cache_rbio(struct btrfs_raid_bio *rbio)
{
struct btrfs_stripe_hash_table *table;
+++++++++++++++++ ++++++++ const unsigned int limit = rbio_cache_limit(rbio->bioc->fs_info);
+++++++++++ ++ ++ ++++++++ bool pin;
if (!test_bit(RBIO_CACHE_READY_BIT, &rbio->flags))
return;
- -
+++++++++++ ++ ++ ++++++++ pin = rbio_needs_pin(rbio);
+++++++++++++++++ ++++++++ /* Even a disabled cache keeps a stripe its run has not finished. */
+++++++++++++++++ ++++++++ if (!limit && !pin)
+++++++++++++++++ ++++++++ return;
+ +
table = rbio->bioc->fs_info->stripe_hash_table;
spin_lock(&table->cache_lock);
spin_unlock(&rbio->bio_list_lock);
----------- -- -- -------- if (table->cache_size > RBIO_CACHE_SIZE) {
----------- -- -- -------- struct btrfs_raid_bio *found;
----------- -- -- --------
----------- -- -- -------- found = list_last_entry(&table->stripe_cache,
----------- -- -- -------- struct btrfs_raid_bio,
----------- -- -- -------- stripe_cache);
----------- -- -- --------
----------- -- -- -------- if (found != rbio)
----------- -- -- -------- __remove_rbio_from_cache(found);
----------- -- -- -------- }
- - rbio_cache_prune(table, rbio, RBIO_CACHE_SIZE);
+++++++++++++++++ ++++++++ /* Prunes as far as needed: the limit can have been lowered. */
+++++++++++++++++ ++++++++ rbio_cache_prune(table, rbio, limit);
spin_unlock(&table->cache_lock);
}
/* No way to verify P/Q as they are not covered by data csum. */
if (stripe_nr >= rbio->nr_data)
return 0;
+++++++ +++ ++ ++ ++++++++
+++++++ ++++++++++++++++++ /* No data checksum for this sector. */
+++++++ ++++++++++++++++++ if (!test_bit(total_sector_nr, rbio->csum_bitmap))
+++++++ ++++++++++++++++++ return 0;
+++++++ ++++++++++++++++++
/*
* If we're rebuilding a read, we have to use pages from the
* bio list if possible.
}
}
--------- - -- -- --------/*
--------- - -- -- -------- * To determine if we need to read any sector from the disk.
--------- - -- -- -------- * Should only be utilized in RMW path, to skip cached rbio.
--------- - -- -- -------- */
--------- - -- -- --------static bool need_read_stripe_sectors(struct btrfs_raid_bio *rbio)
+++++++++++ ++ ++ ++++++++static void rbio_record_pad(struct btrfs_raid_bio *rbio,
+++++++++++ ++ ++ ++++++++ const struct btrfs_stripe_pad_info *info)
+ {
--------- - -- -- -------- int i;
+++++++++++ ++ ++ ++++++++ rbio->pad_reason = info->reason;
+++++++++++ ++ ++ ++++++++ rbio->pad_norun = info->norun;
+++++++++++ ++ ++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
+++++++++++ ++ ++ ++++++++ rbio->pad_run_open = info->run_open;
+++++++++++ ++ ++ ++++++++ rbio->pad_run_class = info->run_class;
+++++++++++ ++ ++ ++++++++ rbio->pad_run_start = info->run_start;
+++++++++++ ++ ++ ++++++++ rbio->pad_run_end = info->run_end;
+++++++++++ ++ ++ ++++++++ rbio->pad_run_offset = info->run_offset;
+++++++++++ ++ ++ ++++++++#endif
+++++++++++ ++ ++ ++++++++}
+
--------- - -- -- -------- for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) {
------- - - -- -- -------- struct sector_ptr *sector = &rbio->stripe_sectors[i];
+++++++++++ ++ ++ ++++++++static_assert(BTRFS_STRIPE_PAD_NR_REASONS + 1 ==
+++++++++++ ++ ++ ++++++++ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_park_stats.data_rmw_by));
+++++++++++ ++ ++ ++++++++static_assert(BTRFS_STRIPE_NORUN_NR ==
+++++++++++ ++ ++ ++++++++ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_park_stats.data_rmw_norun));
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++#ifdef CONFIG_BTRFS_DEBUG
+++++++++++ ++ ++ ++++++++static_assert(BTRFS_STRIPE_PAD_NR_REASONS + 1 ==
+++++++++++ ++ ++ ++++++++ ARRAY_SIZE(((struct btrfs_fs_info *)0)->stripe_rmw_audit_stats.hole_by));
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/* What the audit found under one RMW's uncovered sectors. */
+++++++++++ ++ ++ ++++++++struct rmw_audit {
+++++++++++ ++ ++ ++++++++ int foreign; /* sectors under extents another root owns */
+++++++++++ ++ ++ ++++++++ int reloc; /* sectors under extents only relocation owns */
+++++++++++ ++ ++ ++++++++ u64 min_gen;
+++++++++++ ++ ++ ++++++++ u64 max_gen;
+++++++++++ ++ ++ ++++++++ u64 first; /* first foreign extent */
+++++++++++ ++ ++ ++++++++ u64 first_root; /* one other root referencing it (0: shared ref) */
+++++++++++ ++ ++ ++++++++};
+ +
------- - - -- -- -------- /*
------- - - -- -- -------- * We have a sector which doesn't have page nor uptodate,
------- - - -- -- -------- * thus this rbio can not be cached one, as cached one must
------- - - -- -- -------- * have all its data sectors present and uptodate.
------- - - -- -- -------- */
------- - - -- -- -------- if (!sector->has_paddr || !sector->uptodate)
------- - - -- -- -------- return true;
+++++++++++ ++ ++ ++++++++/* One extent being classified: its uncovered sectors and its owners. */
+++++++++++ ++ ++ ++++++++struct rmw_audit_ext {
+++++++++++ ++ ++ ++++++++ u64 bytenr;
+++++++++++ ++ ++ ++++++++ u64 gen;
+++++++++++ ++ ++ ++++++++ int sectors;
+++++++++++ ++ ++ ++++++++ int reloc_refs;
+++++++++++ ++ ++ ++++++++ int other_refs; /* includes shared refs: owner unknown, so foreign */
+++++++++++ ++ ++ ++++++++ u64 other_root;
+++++++++++ ++ ++ ++++++++};
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++static void rmw_audit_ext_ref(struct rmw_audit_ext *e, u64 root)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ if (root == BTRFS_DATA_RELOC_TREE_OBJECTID) {
+++++++++++ ++ ++ ++++++++ e->reloc_refs++;
+++++++++++ ++ ++ ++++++++ } else {
+++++++++++ ++ ++ ++++++++ e->other_refs++;
+++++++++++ ++ ++ ++++++++ if (!e->other_root)
+++++++++++ ++ ++ ++++++++ e->other_root = root;
+ + }
------- - - -- -- -------- return false;
+ + }
+ +
------- - - -- -- --------static void rmw_rbio(struct btrfs_raid_bio *rbio)
+++++++++++ ++ ++ ++++++++static void rmw_audit_ext_done(struct rmw_audit *a, struct rmw_audit_ext *e)
+ + {
------- - - -- -- -------- struct bio_list bio_list;
------- - - -- -- -------- int sectornr;
------- - - -- -- -------- int ret = 0;
------- - - -- -- --------
------- - - -- -- -------- /*
------- - - -- -- -------- * Allocate the pages for parity first, as P/Q pages will always be
------- - - -- -- -------- * needed for both full-stripe and sub-stripe writes.
------- - - -- -- -------- */
------- - - -- -- -------- ret = alloc_rbio_parity_pages(rbio);
------- - - -- -- -------- if (ret < 0)
+++++++++++ ++ ++ ++++++++ if (!e->sectors)
+ + goto out;
+++++++++++ ++ ++ ++++++++ /* No ref seen at all: be conservative and call it foreign. */
+++++++++++ ++ ++ ++++++++ if (e->other_refs || !e->reloc_refs) {
+++++++++++ ++ ++ ++++++++ if (!a->foreign) {
+++++++++++ ++ ++ ++++++++ a->first = e->bytenr;
+++++++++++ ++ ++ ++++++++ a->first_root = e->other_root;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ a->foreign += e->sectors;
+++++++++++ ++ ++ ++++++++ a->min_gen = min(a->min_gen, e->gen);
+++++++++++ ++ ++ ++++++++ a->max_gen = max(a->max_gen, e->gen);
+++++++++++ ++ ++ ++++++++ } else {
+++++++++++ ++ ++ ++++++++ a->reloc += e->sectors;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++out:
+++++++++++ ++ ++ ++++++++ memset(e, 0, sizeof(*e));
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Classify the data sectors of this rbio's stripe that the write does NOT
+++++++++++ ++ ++ ++++++++ * cover by the COMMITTED extent under them and that extent's owners. Those
+++++++++++ ++ ++ ++++++++ * are the sectors a torn RMW can destroy after a crash. An extent owned only
+++++++++++ ++ ++ ++++++++ * by the data reloc tree is relocation's own orphaned copy, discarded by
+++++++++++ ++ ++ ++++++++ * orphan cleanup if a crash comes before its pointers move; any other owner
+++++++++++ ++ ++ ++++++++ * is a real write hole. Commit root, no locking, as in
+++++++++++ ++ ++ ++++++++ * rmw_block_referenced(). Returns -1 when the lookup cannot answer.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * The commit root is switched shortly before the super lands, so an extent
+++++++++++ ++ ++ ++++++++ * of the transaction committing right now can count early: this errs towards
+++++++++++ ++ ++ ++++++++ * reporting a hole, never towards hiding one.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static int rmw_committed_uncovered(struct btrfs_raid_bio *rbio,
+++++++++++ ++ ++ ++++++++ struct rmw_audit *a)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++ ++ ++ ++++++++ const u64 stripe_start = rbio->bioc->full_stripe_logical;
+++++++++++ ++ ++ ++++++++ const u64 stripe_end = stripe_start +
+++++++++++ ++ ++ ++++++++ (u64)rbio->nr_data * BTRFS_STRIPE_LEN;
+++++++++++ ++ ++ ++++++++ const u32 bits = fs_info->sectorsize_bits;
+++++++++++ ++ ++ ++++++++ struct btrfs_root *extent_root = btrfs_extent_root(fs_info, stripe_start);
+++++++++++ ++ ++ ++++++++ struct rmw_audit_ext e = { 0 };
+++++++++++ ++ ++ ++++++++ struct btrfs_path *path;
+++++++++++ ++ ++ ++++++++ struct btrfs_key key;
+++++++++++ ++ ++ ++++++++ int ret;
+ +
+++++++++++ ++ ++ ++++++++ memset(a, 0, sizeof(*a));
+++++++++++ ++ ++ ++++++++ a->min_gen = U64_MAX;
+ + /*
------- - - -- -- -------- * Either full stripe write, or we have every data sector already
------- - - -- -- -------- * cached, can go to write path immediately.
+++++++++++ ++ ++ ++++++++ * The caller holds the full stripe lock, so nothing here may wait for
+++++++++++ ++ ++ ++++++++ * something that can wait for this stripe: a tree block read that
+++++++++++ ++ ++ ++++++++ * needs a raid56 rebuild of it (mixed block groups keep tree blocks in
+++++++++++ ++ ++ ++++++++ * it, see fill_data_csums(), so they are not audited), or
+++++++++++ ++ ++ ++++++++ * commit_root_sem, whose holder may be reading such a block. Read
+++++++++++ ++ ++ ++++++++ * cached blocks only, and take the semaphore only if it is free. Each
+++++++++++ ++ ++ ++++++++ * miss counts as an audit error.
+ + */
------- - - -- -- -------- if (!rbio_is_full(rbio) && need_read_stripe_sectors(rbio)) {
------- - - -- -- -------- /*
------- - - -- -- -------- * Now we're doing sub-stripe write, also need all data stripes
------- - - -- -- -------- * to do the full RMW.
------- - - -- -- -------- */
------- - - -- -- -------- ret = alloc_rbio_data_pages(rbio);
------- - - -- -- -------- if (ret < 0)
------- - - -- -- -------- goto out;
------- - - -- -- --------
------- - - -- -- -------- index_rbio_pages(rbio);
+++++++++++ ++ ++ ++++++++ if (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_METADATA)
+++++++++++ ++ ++ ++++++++ return -1;
+++++++++++ ++ ++ ++++++++ path = btrfs_alloc_path();
+++++++++++ ++ ++ ++++++++ if (!path)
+++++++++++ ++ ++ ++++++++ return -1;
+++++++++++ ++ ++ ++++++++ path->search_commit_root = 1;
+++++++++++ ++ ++ ++++++++ path->skip_locking = 1;
+++++++++++ ++ ++ ++++++++ path->nowait = 1;
+ +
------- - - -- -- -------- ret = rmw_read_wait_recover(rbio);
+++++++++++ ++ ++ ++++++++ if (!down_read_trylock(&fs_info->commit_root_sem)) {
+++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
+++++++++++ ++ ++ ++++++++ return -1;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ key.objectid = stripe_start;
+++++++++++ ++ ++ ++++++++ key.type = BTRFS_EXTENT_ITEM_KEY;
+++++++++++ ++ ++ ++++++++ key.offset = 0;
+++++++++++ ++ ++ ++++++++ ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+++++++++++ ++ ++ ++++++++ if (ret < 0)
+++++++++++ ++ ++ ++++++++ goto fail;
+++++++++++ ++ ++ ++++++++ /* An extent starting below the stripe may reach into it. */
+++++++++++ ++ ++ ++++++++ ret = btrfs_previous_extent_item(extent_root, path,
+++++++++++ ++ ++ ++++++++ stripe_start > SZ_1G ? stripe_start - SZ_1G : 0);
+++++++++++ ++ ++ ++++++++ if (ret < 0)
+++++++++++ ++ ++ ++++++++ goto fail;
+++++++++++ ++ ++ ++++++++ if (ret > 0) {
+++++++++++ ++ ++ ++++++++ btrfs_release_path(path);
+++++++++++ ++ ++ ++++++++ ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+ + if (ret < 0)
------- - - -- -- -------- goto out;
+++++++++++ ++ ++ ++++++++ goto fail;
+ + }
------- - - -- -- --------
------- - - -- -- -------- /*
------- - - -- -- -------- * At this stage we're not allowed to add any new bios to the
------- - - -- -- -------- * bio list any more, anyone else that wants to change this stripe
------- - - -- -- -------- * needs to do their own rmw.
+++++++++++ ++ ++ ++++++++ while (1) {
+++++++++++ ++ ++ ++++++++ struct extent_buffer *leaf = path->nodes[0];
+++++++++++ ++ ++ ++++++++ const int slot = path->slots[0];
+++++++++++ ++ ++ ++++++++ struct btrfs_extent_item *ei;
+++++++++++ ++ ++ ++++++++ unsigned long ptr, end;
+++++++++++ ++ ++ ++++++++ u64 lo, hi;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (slot >= btrfs_header_nritems(leaf)) {
+++++++++++ ++ ++ ++++++++ ret = btrfs_next_leaf(extent_root, path);
+++++++++++ ++ ++ ++++++++ if (ret < 0)
+++++++++++ ++ ++ ++++++++ goto fail;
+++++++++++ ++ ++ ++++++++ if (ret > 0)
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ btrfs_item_key_to_cpu(leaf, &key, slot);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Keyed backrefs of the extent being classified. */
+++++++++++ ++ ++ ++++++++ if (e.sectors && key.objectid == e.bytenr) {
+++++++++++ ++ ++ ++++++++ if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
+++++++++++ ++ ++ ++++++++ struct btrfs_extent_data_ref *dref;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ dref = btrfs_item_ptr(leaf, slot,
+++++++++++ ++ ++ ++++++++ struct btrfs_extent_data_ref);
+++++++++++ ++ ++ ++++++++ rmw_audit_ext_ref(&e, btrfs_extent_data_ref_root(leaf, dref));
+++++++++++ ++ ++ ++++++++ } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
+++++++++++ ++ ++ ++++++++ e.other_refs++;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ path->slots[0]++;
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (key.type != BTRFS_EXTENT_ITEM_KEY) {
+++++++++++ ++ ++ ++++++++ if (key.objectid >= stripe_end)
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ path->slots[0]++;
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ rmw_audit_ext_done(a, &e);
+++++++++++ ++ ++ ++++++++ if (key.objectid >= stripe_end)
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ if (key.objectid + key.offset <= stripe_start) {
+++++++++++ ++ ++ ++++++++ path->slots[0]++;
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item);
+++++++++++ ++ ++ ++++++++ lo = max(key.objectid, stripe_start);
+++++++++++ ++ ++ ++++++++ hi = min(key.objectid + key.offset, stripe_end);
+++++++++++ ++ ++ ++++++++ for (u64 b = lo; b < hi; b += fs_info->sectorsize)
+++++++++++ ++ ++ ++++++++ if (!rbio->bio_sectors[(b - stripe_start) >> bits].has_paddr)
+++++++++++ ++ ++ ++++++++ e.sectors++;
+++++++++++ ++ ++ ++++++++ if (!e.sectors) {
+++++++++++ ++ ++ ++++++++ path->slots[0]++;
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ e.bytenr = key.objectid;
+++++++++++ ++ ++ ++++++++ e.gen = btrfs_extent_generation(leaf, ei);
+++++++++++ ++ ++ ++++++++ /* Inline backrefs: data extents carry no tree_block_info. */
+++++++++++ ++ ++ ++++++++ ptr = (unsigned long)(ei + 1);
+++++++++++ ++ ++ ++++++++ end = (unsigned long)ei + btrfs_item_size(leaf, slot);
+++++++++++ ++ ++ ++++++++ while (ptr < end) {
+++++++++++ ++ ++ ++++++++ struct btrfs_extent_inline_ref *iref = (void *)ptr;
+++++++++++ ++ ++ ++++++++ const int type = btrfs_extent_inline_ref_type(leaf, iref);
+++++++++++ ++ ++ ++++++++ const u32 size = btrfs_extent_inline_ref_size(type);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (type == BTRFS_EXTENT_DATA_REF_KEY) {
+++++++++++ ++ ++ ++++++++ struct btrfs_extent_data_ref *dref =
+++++++++++ ++ ++ ++++++++ (void *)&iref->offset;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ rmw_audit_ext_ref(&e, btrfs_extent_data_ref_root(leaf, dref));
+++++++++++ ++ ++ ++++++++ } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
+++++++++++ ++ ++ ++++++++ e.other_refs++;
+++++++++++ ++ ++ ++++++++ } else if (type != BTRFS_EXTENT_OWNER_REF_KEY) {
+++++++++++ ++ ++ ++++++++ /* Not a data ref we understand: foreign. */
+++++++++++ ++ ++ ++++++++ e.other_refs++;
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (!size)
+++++++++++ ++ ++ ++++++++ break;
+++++++++++ ++ ++ ++++++++ ptr += size;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ path->slots[0]++;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ rmw_audit_ext_done(a, &e);
+++++++++++ ++ ++ ++++++++ up_read(&fs_info->commit_root_sem);
+++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
+++++++++++ ++ ++ ++++++++ return a->foreign + a->reloc;
+++++++++++ ++ ++ ++++++++fail:
+++++++++++ ++ ++ ++++++++ up_read(&fs_info->commit_root_sem);
+++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
+++++++++++ ++ ++ ++++++++ return -1;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * With /sys/fs/btrfs/<uuid>/debug/stripe_rmw_audit set: prove from the
+++++++++++ ++ ++ ++++++++ * committed extent tree, not from the allocator's design, that a data RMW put
+++++++++++ ++ ++ ++++++++ * no other owner's committed data at risk, and log the writer of any that
+++++++++++ ++ ++ ++++++++ * did (and of every NO_RUN RMW, the class the allocator cannot explain by
+++++++++++ ++ ++ ++++++++ * construction). @idx is the RMW's data_rmw_by index.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void rbio_audit_rmw(struct btrfs_raid_bio *rbio, u8 idx)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++ ++ ++ ++++++++ const u64 stripe_start = rbio->bioc->full_stripe_logical;
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ struct rmw_audit a;
+++++++++++ ++ ++ ++++++++ u64 root_id = 0, ino = 0, foff = 0, oflags = 0, oseq = 0, odisk = 0;
+++++++++++ ++ ++ ++++++++ u64 bg_flags = 0;
+++++++++++ ++ ++ ++++++++ bool orun = false;
+++++++++++ ++ ++ ++++++++ int nbios = 0;
+++++++++++ ++ ++ ++++++++ int ro = -1;
+++++++++++ ++ ++ ++++++++ int ret;
+++++++++++ ++ ++ ++++++++ struct bio *bio;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!READ_ONCE(fs_info->stripe_rmw_audit))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.audited);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ spin_lock(&rbio->bio_list_lock);
+++++++++++ ++ ++ ++++++++ bio_list_for_each(bio, &rbio->bio_list) {
+++++++++++ ++ ++ ++++++++ struct btrfs_bio *bbio = btrfs_bio(bio);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (nbios++)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (!bbio->inode ||
+++++++++++ ++ ++ ++++++++ btrfs_ino(bbio->inode) == BTRFS_BTREE_INODE_OBJECTID)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ root_id = btrfs_root_id(bbio->inode->root);
+++++++++++ ++ ++ ++++++++ ino = btrfs_ino(bbio->inode);
+++++++++++ ++ ++ ++++++++ foff = bbio->file_offset;
+++++++++++ ++ ++ ++++++++ if (bbio->ordered) {
+++++++++++ ++ ++ ++++++++ oflags = bbio->ordered->flags;
+++++++++++ ++ ++ ++++++++ oseq = bbio->ordered->stripe_seq;
+++++++++++ ++ ++ ++++++++ odisk = bbio->ordered->disk_bytenr;
+++++++++++ ++ ++ ++++++++ orun = bbio->ordered->stripe_run != NULL;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ spin_unlock(&rbio->bio_list_lock);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* The audit needs bio sectors mapped; idempotent. */
+++++++++++ ++ ++ ++++++++ index_rbio_pages(rbio);
+++++++++++ ++ ++ ++++++++ ret = rmw_committed_uncovered(rbio, &a);
+++++++++++ ++ ++ ++++++++ if (ret < 0) {
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.audit_err);
+++++++++++ ++ ++ ++++++++ } else if (a.foreign) {
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.hole_by[idx]);
+++++++++++ ++ ++ ++++++++ atomic64_add(a.foreign, &fs_info->stripe_rmw_audit_stats.hole_sectors);
+++++++++++ ++ ++ ++++++++ if (root_id == BTRFS_DATA_RELOC_TREE_OBJECTID)
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.reloc_writer_foreign);
+++++++++++ ++ ++ ++++++++ } else if (a.reloc) {
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_rmw_audit_stats.hole_reloc_only);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (ret > 0)
+++++++++++ ++ ++ ++++++++ atomic64_add(a.reloc, &fs_info->stripe_rmw_audit_stats.hole_reloc_sectors);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (idx == BTRFS_STRIPE_PAD_NO_RUN) {
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+++++++++++ ++ ++ ++++++++ if (bg) {
+++++++++++ ++ ++ ++++++++ ro = data_race(bg->ro);
+++++++++++ ++ ++ ++++++++ bg_flags = bg->flags;
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Separate call sites, so separate rate limits: a flood of NO_RUN
+++++++++++ ++ ++ ++++++++ * lines cannot starve the hole lines.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (ret > 0 && a.foreign)
+++++++++++ ++ ++ ++++++++ btrfs_warn_rl(fs_info,
+++++++++++ ++ ++ ++++++++"rmw audit: write hole: stripe %llu reason %u norun %u foreign %d reloc %d gen %llu..%llu first %llu first_root %lld fsgen %llu committed %llu writer root %lld ino %llu foff %llu oe_flags 0x%llx oe_disk %llu oe_seq %llu oe_run %d nbios %d",
+++++++++++ ++ ++ ++++++++ stripe_start, idx, rbio->pad_norun, a.foreign,
+++++++++++ ++ ++ ++++++++ a.reloc, a.min_gen, a.max_gen, a.first,
+++++++++++ ++ ++ ++++++++ (s64)a.first_root, btrfs_get_fs_generation(fs_info),
+++++++++++ ++ ++ ++++++++ btrfs_get_last_trans_committed(fs_info),
+++++++++++ ++ ++ ++++++++ (s64)root_id, ino, foff, oflags, odisk, oseq, orun,
+++++++++++ ++ ++ ++++++++ nbios);
+++++++++++ ++ ++ ++++++++ else if (idx == BTRFS_STRIPE_PAD_NO_RUN)
+++++++++++ ++ ++ ++++++++ btrfs_info_rl(fs_info,
+++++++++++ ++ ++ ++++++++"rmw audit: no_run RMW stripe %llu norun %u bg_ro %d bg_flags 0x%llx run %llu..%llu off %llu open %u class %u reloc_sectors %d fsgen %llu committed %llu writer root %lld ino %llu foff %llu oe_flags 0x%llx oe_seq %llu oe_run %d nbios %d",
+++++++++++ ++ ++ ++++++++ stripe_start, rbio->pad_norun, ro, bg_flags,
+++++++++++ ++ ++ ++++++++ rbio->pad_run_start, rbio->pad_run_end,
+++++++++++ ++ ++ ++++++++ rbio->pad_run_offset, rbio->pad_run_open,
+++++++++++ ++ ++ ++++++++ rbio->pad_run_class, ret > 0 ? a.reloc : 0,
+++++++++++ ++ ++ ++++++++ btrfs_get_fs_generation(fs_info),
+++++++++++ ++ ++ ++++++++ btrfs_get_last_trans_committed(fs_info),
+++++++++++ ++ ++ ++++++++ (s64)root_id, ino, foff, oflags, oseq, orun, nbios);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++#else
+++++++++++ ++ ++ ++++++++static void rbio_audit_rmw(struct btrfs_raid_bio *rbio, u8 idx) { }
+++++++++++ ++ ++ ++++++++#endif
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Attribute a read-modify-write to why padding declined for this rbio. With
+++++++++++ ++ ++ ++++++++ * allow_rmw empty, a data RMW in a stripe_alloc group is expected in two
+++++++++++ ++ ++ ++++++++ * places only: at the open run's own frontier (UNCOVERED: the sectors it
+++++++++++ ++ ++ ++++++++ * reads are the same run's, in the same transaction, not yet durable), and
+++++++++++ ++ ++ ++++++++ * from relocation writing into its own preallocated extents, which nothing
+++++++++++ ++ ++ ++++++++ * but the data reloc inode references until relocation moves the pointers
+++++++++++ ++ ++ ++++++++ * after writeback. Any other data RMW in a covered group is unexplained.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void rbio_count_rmw_reason(struct btrfs_raid_bio *rbio)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++ ++ ++ ++++++++ u8 idx = rbio->pad_reason;
+++++++++++ ++ ++ ++++++++ bool reloc_writer = false;
+++++++++++ ++ ++ ++++++++ struct bio *bio;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (idx >= BTRFS_STRIPE_PAD_NR_REASONS)
+++++++++++ ++ ++ ++++++++ idx = BTRFS_STRIPE_PAD_NR_REASONS; /* not tried */
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_by[idx]);
+++++++++++ ++ ++ ++++++++ if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (idx == BTRFS_STRIPE_PAD_NO_RUN) {
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (rbio->pad_norun < BTRFS_STRIPE_NORUN_NR)
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_norun[rbio->pad_norun]);
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info,
+++++++++++ ++ ++ ++++++++ rbio->bioc->full_stripe_logical);
+++++++++++ ++ ++ ++++++++ if (bg) {
+++++++++++ ++ ++ ++++++++ if (data_race(bg->ro))
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_norun_ro);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* The writer: the first bio's inode. */
+++++++++++ ++ ++ ++++++++ spin_lock(&rbio->bio_list_lock);
+++++++++++ ++ ++ ++++++++ bio = bio_list_peek(&rbio->bio_list);
+++++++++++ ++ ++ ++++++++ if (bio && btrfs_bio(bio)->inode)
+++++++++++ ++ ++ ++++++++ reloc_writer = btrfs_is_data_reloc_root(btrfs_bio(bio)->inode->root);
+++++++++++ ++ ++ ++++++++ spin_unlock(&rbio->bio_list_lock);
+++++++++++ ++ ++ ++++++++ /* Disjoint from the frontier count, so the two can be summed. */
+++++++++++ ++ ++ ++++++++ if (reloc_writer && idx != BTRFS_STRIPE_PAD_UNCOVERED)
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw_reloc_writer);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ rbio_audit_rmw(rbio, idx);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Try to avoid the RMW read phase of a sub-stripe write by zero-filling the
+++++++++++ ++ ++ ++++++++ * uncovered data sectors. Legal only where the stripe's run says a sector
+++++++++++ ++ ++ ++++++++ * is dead: never allocated, handed back, or allocated with its write still
+++++++++++ ++ ++ ++++++++ * to come. That write, like any allocation the run makes after this check,
+++++++++++ ++ ++ ++++++++ * serializes behind this rbio's stripe lock and lands over the zeros.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static bool rmw_try_pad_full(struct btrfs_raid_bio *rbio)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++ ++ ++ ++++++++ const u64 stripe_start = rbio->bioc->full_stripe_logical;
+++++++++++ ++ ++ ++++++++ const u64 stripe_len = (u64)rbio->nr_data * BTRFS_STRIPE_LEN;
+++++++++++ ++ ++ ++++++++ const u32 sectorsize = fs_info->sectorsize;
+++++++++++ ++ ++ ++++++++ const u32 nsectors = rbio->nr_data * rbio->stripe_nsectors;
+++++++++++ ++ ++ ++++++++ struct btrfs_stripe_pad_info info;
+++++++++++ ++ ++ ++++++++ unsigned long *dead;
+++++++++++ ++ ++ ++++++++ unsigned long *unwritten;
+++++++++++ ++ ++ ++++++++ u32 covered = 0;
+++++++++++ ++ ++ ++++++++ u32 nr_unwritten = 0;
+++++++++++ ++ ++ ++++++++ bool ok = false;
+++++++++++ ++ ++ ++++++++ int i;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ rbio->pad_reason = RBIO_PAD_NOT_TRIED;
+++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ dead = bitmap_zalloc(nsectors, GFP_NOFS);
+++++++++++ ++ ++ ++++++++ unwritten = bitmap_zalloc(nsectors, GFP_NOFS);
+++++++++++ ++ ++ ++++++++ if (!dead || !unwritten) {
+++++++++++ ++ ++ ++++++++ bitmap_free(dead);
+++++++++++ ++ ++ ++++++++ bitmap_free(unwritten);
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (!btrfs_stripe_run_pad_mask(fs_info, stripe_start, stripe_len, dead,
+++++++++++ ++ ++ ++++++++ unwritten, &info)) {
+++++++++++ ++ ++ ++++++++ rbio_record_pad(rbio, &info);
+++++++++++ ++ ++ ++++++++ trace_btrfs_stripe_pad_decline(fs_info, stripe_start,
+++++++++++ ++ ++ ++++++++ stripe_len, 0, 0, nsectors,
+++++++++++ ++ ++ ++++++++ info.run_start, info.run_end,
+++++++++++ ++ ++ ++++++++ info.run_offset,
+++++++++++ ++ ++ ++++++++ info.inflight_bytes,
+++++++++++ ++ ++ ++++++++ info.reason);
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Populate bio_sectors so covered sectors are visible below. */
+++++++++++ ++ ++ ++++++++ index_rbio_pages(rbio);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ for (i = 0; i < nsectors; i++)
+++++++++++ ++ ++ ++++++++ if (rbio->bio_sectors[i].has_paddr)
+++++++++++ ++ ++ ++++++++ covered++;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Every uncovered sector must be dead per the run. A live uncovered
+++++++++++ ++ ++ ++++++++ * sector that was already written holds data this write does not
+++++++++++ ++ ++ ++++++++ * carry, so refuse (UNCOVERED); the stripe pin (rbio_needs_pin())
+++++++++++ ++ ++ ++++++++ * normally supplies such sectors from the cache, so nothing is read.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ for (i = 0; i < nsectors; i++) {
+++++++++++ ++ ++ ++++++++ if (rbio->bio_sectors[i].has_paddr)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (!test_bit(i, dead)) {
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_uncovered);
+++++++++++ ++ ++ ++++++++ info.reason = BTRFS_STRIPE_PAD_UNCOVERED;
+++++++++++ ++ ++ ++++++++ rbio_record_pad(rbio, &info);
+++++++++++ ++ ++ ++++++++ trace_btrfs_stripe_pad_decline(fs_info, stripe_start,
+++++++++++ ++ ++ ++++++++ stripe_len, 0, covered,
+++++++++++ ++ ++ ++++++++ nsectors,
+++++++++++ ++ ++ ++++++++ info.run_start,
+++++++++++ ++ ++ ++++++++ info.run_end,
+++++++++++ ++ ++ ++++++++ info.run_offset,
+++++++++++ ++ ++ ++++++++ info.inflight_bytes,
+++++++++++ ++ ++ ++++++++ info.reason);
+++++++++++ ++ ++ ++++++++ goto out;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (test_bit(i, unwritten))
+++++++++++ ++ ++ ++++++++ nr_unwritten++;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ ok = true;
+++++++++++ ++ ++ ++++++++out:
+++++++++++ ++ ++ ++++++++ bitmap_free(dead);
+++++++++++ ++ ++ ++++++++ bitmap_free(unwritten);
+++++++++++ ++ ++ ++++++++ if (!ok)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* The oracle agreed: only a page allocation failure refuses now. */
+++++++++++ ++ ++ ++++++++ rbio->pad_reason = BTRFS_STRIPE_PAD_OK;
+++++++++++ ++ ++ ++++++++ if (alloc_rbio_data_pages(rbio) < 0)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) {
+++++++++++ ++ ++ ++++++++ struct sector_ptr *sector;
+++++++++++ ++ ++ ++++++++ void *kaddr;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (rbio->bio_sectors[i].has_paddr)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ sector = &rbio->stripe_sectors[i];
+++++++++++ ++ ++ ++++++++ kaddr = kmap_local_sector(sector);
+++++++++++ ++ ++ ++++++++ memset(kaddr, 0, sectorsize);
+++++++++++ ++ ++ ++++++++ kunmap_local(kaddr);
+++++++++++ ++ ++ ++++++++ sector->uptodate = true;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ set_bit(RBIO_PADDED_BIT, &rbio->flags);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * The whole stripe goes out: every vertical position is written.
+++++++++++ ++ ++ ++++++++ * Under bio_list_lock: until RBIO_RMW_LOCKED_BIT is set, merge_rbio()
+++++++++++ ++ ++ ++++++++ * can still OR another rbio's bits into this bitmap.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ spin_lock(&rbio->bio_list_lock);
+++++++++++ ++ ++ ++++++++ bitmap_set(&rbio->dbitmap, 0, rbio->stripe_nsectors);
+++++++++++ ++ ++ ++++++++ spin_unlock(&rbio->bio_list_lock);
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.padded);
+++++++++++ ++ ++ ++++++++ if (nr_unwritten)
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.padded_unwritten);
+++++++++++ ++ ++ ++++++++ return true;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++ + ++ ++ ++++++++/*
+++++++++ + ++ ++ ++++++++ * To determine if we need to read any sector from the disk.
+++++++++ + ++ ++ ++++++++ * Should only be utilized in RMW path, to skip cached rbio.
+++++++++ + ++ ++ ++++++++ */
+++++++++ + ++ ++ ++++++++static bool need_read_stripe_sectors(struct btrfs_raid_bio *rbio)
+++++++++ + ++ ++ ++++++++{
+++++++++ + ++ ++ ++++++++ int i;
+++++++++ + ++ ++ ++++++++
+++++++++ + ++ ++ ++++++++ for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) {
+++++++ + + ++ ++ ++++++++ struct sector_ptr *sector = &rbio->stripe_sectors[i];
+++++++ + + ++ ++ ++++++++
+++++++ + + ++ ++ ++++++++ /*
+++++++ + + ++ ++ ++++++++ * We have a sector which doesn't have page nor uptodate,
+++++++ + + ++ ++ ++++++++ * thus this rbio can not be cached one, as cached one must
+++++++ + + ++ ++ ++++++++ * have all its data sectors present and uptodate.
+++++++ + + ++ ++ ++++++++ */
+++++++ + + ++ ++ ++++++++ if (!sector->has_paddr || !sector->uptodate)
+++++++ + + ++ ++ ++++++++ return true;
+++++++ + + ++ ++ ++++++++ }
+++++++ + + ++ ++ ++++++++ return false;
+++++++ + + ++ ++ ++++++++}
+++++++ + + ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Read the header of the tree block at data-stripe sector @i, if there is
+++++++++++ ++ ++ ++++++++ * one, from whichever copy of the sector this rbio is holding.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * @from_bio picks the source: the caller's pages (what this write is putting
+++++++++++ ++ ++ ++++++++ * there) or the stripe pages (what an RMW read found on the disk). Returns
+++++++++++ ++ ++ ++++++++ * false when the sector holds no tree block that belongs at this address --
+++++++++++ ++ ++ ++++++++ * free space, or the remains of one from a previous life of the stripe.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static bool rmw_peek_eb_generation(struct btrfs_raid_bio *rbio, u32 i,
+++++++++++ ++ ++ ++++++++ bool from_bio, u64 *generation)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++ ++ ++ ++++++++ const int stripe_nr = i / rbio->stripe_nsectors;
+++++++++++ ++ ++ ++++++++ const int sectornr = i % rbio->stripe_nsectors;
+++++++++++ ++ ++ ++++++++ const u64 logical = rbio->bioc->full_stripe_logical +
+++++++++++ ++ ++ ++++++++ (u64)stripe_nr * BTRFS_STRIPE_LEN +
+++++++++++ ++ ++ ++++++++ (u64)sectornr * fs_info->sectorsize;
+++++++++++ ++ ++ ++++++++ const struct sector_ptr *sector = from_bio ? &rbio->bio_sectors[i]
+++++++++++ ++ ++ ++++++++ : &rbio->stripe_sectors[i];
+++++++++++ ++ ++ ++++++++ struct btrfs_header *hdr;
+++++++++++ ++ ++ ++++++++ u64 found_bytenr;
+++++++++++ ++ ++ ++++++++ bool bad_fsid;
+++++++++++ ++ ++ ++++++++ void *kaddr;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!sector->has_paddr)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++ if (!from_bio && !sector->uptodate)
+++++++++++ ++ ++ ++++++++ return false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ kaddr = kmap_local_sector(sector);
+++++++++++ ++ ++ ++++++++ hdr = kaddr;
+++++++++++ ++ ++ ++++++++ found_bytenr = btrfs_stack_header_bytenr(hdr);
+++++++++++ ++ ++ ++++++++ *generation = btrfs_stack_header_generation(hdr);
+++++++++++ ++ ++ ++++++++ bad_fsid = memcmp(hdr->fsid, fs_info->fs_devices->metadata_uuid,
+++++++++++ ++ ++ ++++++++ BTRFS_FSID_SIZE) != 0;
+++++++++++ ++ ++ ++++++++ kunmap_local(kaddr);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ return found_bytenr == logical && !bad_fsid;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Is @bytenr a tree block the committed extent tree still references? Read
+++++++++++ ++ ++ ++++++++ * the commit root without locking, as the caching thread does, so this is
+++++++++++ ++ ++ ++++++++ * safe from the raid56 worker with no transaction to join.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Returns 1 if it is (or the lookup failed), 0 if not, and -EAGAIN if the
+++++++++++ ++ ++ ++++++++ * answer needs a tree block that is not in memory. The caller holds the
+++++++++++ ++ ++ ++++++++ * full stripe lock, so it must not read from disk: a read of a block in
+++++++++++ ++ ++ ++++++++ * this stripe that needs reconstruction would wait for that lock.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static int rmw_block_referenced(struct btrfs_fs_info *fs_info, u64 bytenr,
+++++++++++ ++ ++ ++++++++ u64 header_gen, u64 *refs, u64 *extent_gen)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_root *extent_root = btrfs_extent_root(fs_info, bytenr);
+++++++++++ ++ ++ ++++++++ struct btrfs_path *path;
+++++++++++ ++ ++ ++++++++ struct btrfs_key key;
+++++++++++ ++ ++ ++++++++ int live = 1;
+++++++++++ ++ ++ ++++++++ int ret;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ *refs = 0;
+++++++++++ ++ ++ ++++++++ *extent_gen = 0;
+++++++++++ ++ ++ ++++++++ path = btrfs_alloc_path();
+++++++++++ ++ ++ ++++++++ if (!path)
+++++++++++ ++ ++ ++++++++ return 1;
+++++++++++ ++ ++ ++++++++ path->search_commit_root = 1;
+++++++++++ ++ ++ ++++++++ path->skip_locking = 1;
+++++++++++ ++ ++ ++++++++ path->nowait = 1;
+++++++++++ ++ ++ ++++++++ key.objectid = bytenr;
+++++++++++ ++ ++ ++++++++ key.type = BTRFS_METADATA_ITEM_KEY;
+++++++++++ ++ ++ ++++++++ key.offset = (u64)-1;
+++++++++++ ++ ++ ++++++++ if (!down_read_trylock(&fs_info->commit_root_sem)) {
+++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
+++++++++++ ++ ++ ++++++++ return -EAGAIN;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
+++++++++++ ++ ++ ++++++++ if (ret > 0)
+++++++++++ ++ ++ ++++++++ ret = btrfs_previous_extent_item(extent_root, path, bytenr);
+++++++++++ ++ ++ ++++++++ if (ret == 0) {
+++++++++++ ++ ++ ++++++++ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
+++++++++++ ++ ++ ++++++++ if (key.objectid == bytenr &&
+++++++++++ ++ ++ ++++++++ (key.type == BTRFS_METADATA_ITEM_KEY ||
+++++++++++ ++ ++ ++++++++ key.type == BTRFS_EXTENT_ITEM_KEY)) {
+++++++++++ ++ ++ ++++++++ struct btrfs_extent_item *ei;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
+++++++++++ ++ ++ ++++++++ struct btrfs_extent_item);
+++++++++++ ++ ++ ++++++++ *refs = btrfs_extent_refs(path->nodes[0], ei);
+++++++++++ ++ ++ ++++++++ *extent_gen = btrfs_extent_generation(path->nodes[0], ei);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * An extent item at this bytenr from a LATER
+++++++++++ ++ ++ ++++++++ * transaction than the header on disk belongs to a
+++++++++++ ++ ++ ++++++++ * new block allocated here whose write has not landed
+++++++++++ ++ ++ ++++++++ * yet; the header is the dead previous occupant.
+++++++++++ ++ ++ ++++++++ * Only an item of the header's own generation makes
+++++++++++ ++ ++ ++++++++ * the block on disk live.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ live = (*refs != 0 && *extent_gen == header_gen);
+++++++++++ ++ ++ ++++++++ } else {
+++++++++++ ++ ++ ++++++++ live = 0;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ } else if (ret > 0) {
+++++++++++ ++ ++ ++++++++ live = 0;
+++++++++++ ++ ++ ++++++++ } else if (ret == -EAGAIN) {
+++++++++++ ++ ++ ++++++++ live = -EAGAIN;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ up_read(&fs_info->commit_root_sem);
+++++++++++ ++ ++ ++++++++ btrfs_free_path(path);
+++++++++++ ++ ++ ++++++++ return live;
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Decide whether an uncovered metadata read-modify-write is a write hole or
+++++++++++ ++ ++ ++++++++ * only a cost, from the tree blocks it read: their parity is about to be
+++++++++++ ++ ++ ++++++++ * rewritten while their own sectors stay put.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * - Same generation as the blocks this write carries (meta_rmw_cur): the
+++++++++++ ++ ++ ++++++++ * block belongs to the running transaction, which a crash discards --
+++++++++++ ++ ++ ++++++++ * except a log tree block, which is durable once its fsync returns.
+++++++++++ ++ ++ ++++++++ * The audit does not tell log blocks apart, so it cannot see a write
+++++++++++ ++ ++ ++++++++ * hole over fsynced log blocks; keeping them out of shared stripes is
+++++++++++ ++ ++ ++++++++ * stripe_log_run's job.
+++++++++++ ++ ++ ++++++++ * - Older, and still referenced by the committed extent tree: a write hole
+++++++++++ ++ ++ ++++++++ * over committed metadata (meta_rmw_old).
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Comparing against this rbio's own blocks, not fs_info's committed
+++++++++++ ++ ++ ++++++++ * generation, keeps a concurrent commit from skewing the verdict.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void audit_uncovered_rmw(struct btrfs_raid_bio *rbio)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++ ++ ++ ++++++++ const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits;
+++++++++++ ++ ++ ++++++++ const u32 nsectors = rbio->nr_data * rbio->stripe_nsectors;
+++++++++++ ++ ++ ++++++++ u64 write_gen = 0;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Tree blocks only: metadata and system groups. A mixed group's
+++++++++++ ++ ++ ++++++++ * sectors need not hold tree blocks.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ if (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ if (!(rbio->bioc->map_type & (BTRFS_BLOCK_GROUP_METADATA |
+++++++++++ ++ ++ ++++++++ BTRFS_BLOCK_GROUP_SYSTEM)))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Both loops read bio_sectors to tell what this write covers, and the
+++++++++++ ++ ++ ++++++++ * paths that reach here do not all populate it (a cached rbio that
+++++++++++ ++ ++ ++++++++ * failed to pad never indexes). Idempotent, so just do it.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ index_rbio_pages(rbio);
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* The generation this write is putting into the stripe. */
+++++++++++ ++ ++ ++++++++ for (u32 i = 0; i < nsectors; i += sectors_per_tree) {
+++++++++++ ++ ++ ++++++++ u64 gen;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (rmw_peek_eb_generation(rbio, i, true, &gen))
+++++++++++ ++ ++ ++++++++ write_gen = max(write_gen, gen);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (!write_gen)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ for (u32 i = 0; i < nsectors; i += sectors_per_tree) {
+++++++++++ ++ ++ ++++++++ u64 gen;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Covered by this write: it is what we are putting there. */
+++++++++++ ++ ++ ++++++++ if (rbio->bio_sectors[i].has_paddr)
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ if (!rmw_peek_eb_generation(rbio, i, false, &gen)) {
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_free);
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (gen >= write_gen) {
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_cur);
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * An older header is a write hole only if the committed extent
+++++++++++ ++ ++ ++++++++ * tree still references the block: a reclaimed stripe keeps the
+++++++++++ ++ ++ ++++++++ * headers of dead blocks, and tearing those harms nothing a
+++++++++++ ++ ++ ++++++++ * reader can reach. The commit root is what a crash would see.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ {
+++++++++++ ++ ++ ++++++++ const int stripe_nr = i / rbio->stripe_nsectors;
+++++++++++ ++ ++ ++++++++ const int sectornr = i % rbio->stripe_nsectors;
+++++++++++ ++ ++ ++++++++ const u64 logical = rbio->bioc->full_stripe_logical +
+++++++++++ ++ ++ ++++++++ (u64)stripe_nr * BTRFS_STRIPE_LEN +
+++++++++++ ++ ++ ++++++++ (u64)sectornr * fs_info->sectorsize;
+++++++++++ ++ ++ ++++++++ u64 refs;
+++++++++++ ++ ++ ++++++++ u64 egen;
+++++++++++ ++ ++ ++++++++ int ret;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ ret = rmw_block_referenced(fs_info, logical, gen, &refs,
+++++++++++ ++ ++ ++++++++ &egen);
+++++++++++ ++ ++ ++++++++ if (ret == -EAGAIN) {
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_unknown);
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (!ret) {
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_ghost);
+++++++++++ ++ ++ ++++++++ continue;
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_old);
+++++++++++ ++ ++ ++++++++ btrfs_warn_rl(fs_info,
+++++++++++ ++ ++ ++++++++"read-modify-write of stripe %llu rewrites parity over committed tree block %llu at generation %llu (refs %llu, extent generation %llu) while writing generation %llu: write hole",
+++++++++++ ++ ++ ++++++++ rbio->bioc->full_stripe_logical, logical,
+++++++++++ ++ ++ ++++++++ gen, refs, egen, write_gen);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++/*
+++++++++++ ++ ++ ++++++++ * Count and audit a metadata or system read-modify-write
+++++++++++ ++ ++ ++++++++ * (audit_uncovered_rmw()), and warn if it lands in a raid56 group that
+++++++++++ ++ ++ ++++++++ * stripe_meta does not cover. Silent when stripe_alloc is off.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++static void report_uncovered_rmw(struct btrfs_raid_bio *rbio)
+++++++++++ ++ ++ ++++++++{
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++ ++ ++ ++++++++ struct btrfs_block_group *bg;
+++++++++++ ++ ++ ++++++++ bool covered = false;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ /* Data groups are btrfs_is_stripe_alloc_bg()'s. */
+++++++++++ ++ ++ ++++++++ if ((rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA) &&
+++++++++++ ++ ++ ++++++++ !(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_METADATA))
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw);
+++++++++++ ++ ++ ++++++++ audit_uncovered_rmw(rbio);
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Every raid56 metadata and system group is a stripe_meta group while
+++++++++++ ++ ++ ++++++++ * stripe_alloc is on, and stripe_alloc refuses to enable on mixed
+++++++++++ ++ ++ ++++++++ * groups (btrfs_stripe_alloc_check_support()), so this warning is a
+++++++++++ ++ ++ ++++++++ * defensive check: it fires only if a tree block group that should be
+++++++++++ ++ ++ ++++++++ * covered is not.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ bg = btrfs_lookup_block_group(fs_info, rbio->bioc->full_stripe_logical);
+++++++++++ ++ ++ ++++++++ if (bg) {
+++++++++++ ++ ++ ++++++++ covered = btrfs_is_stripe_meta_bg(bg);
+++++++++++ ++ ++ ++++++++ btrfs_put_block_group(bg);
+++++++++++ ++ ++ ++++++++ }
+++++++++++ ++ ++ ++++++++ if (covered)
+++++++++++ ++ ++ ++++++++ return;
+++++++++++ ++ ++ ++++++++ btrfs_warn_rl(fs_info,
+++++++++++ ++ ++ ++++++++"read-modify-write of full stripe %llu: this block group is not covered by stripe_alloc, the raid56 write hole applies to it",
+++++++++++ ++ ++ ++++++++ rbio->bioc->full_stripe_logical);
+++++++++++ ++ ++ ++++++++}
+++++++++++ ++ ++ ++++++++
+++++++ ++++++++++++++++++/*
+++++++ ++++++++++++++++++ * Return true if the rebuild of data sector @sector_nr of @stripe_nr can be
+++++++ ++++++++++++++++++ * trusted enough to write it back. It can when the sector has a csum: the
+++++++ ++++++++++++++++++ * rebuild was verified. Without a csum, it can only when another data sector
+++++++ ++++++++++++++++++ * of the same vertical stripe also failed and has a csum: the two were
+++++++ ++++++++++++++++++ * rebuilt together from P and Q (on RAID6 that is also how a stale sector
+++++++ ++++++++++++++++++ * without a csum is found, see recover_vertical_retry()), and the csum that
+++++++ ++++++++++++++++++ * verified one verifies the other. A lone rebuild without a csum came from
+++++++ ++++++++++++++++++ * parity that nothing checked, and the copy on the disk may still be right.
+++++++ ++++++++++++++++++ */
+++++++ ++++++++++++++++++static bool rmw_repair_verified(struct btrfs_raid_bio *rbio, int stripe_nr,
+++++++ ++++++++++++++++++ int sector_nr)
+++++++ ++++++++++++++++++{
+++++++ ++++++++++++++++++ if (!rbio->csum_bitmap)
+++++++ ++++++++++++++++++ return false;
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++ for (int i = 0; i < rbio->nr_data; i++) {
+++++++ ++++++++++++++++++ const int index = rbio_stripe_sector_index(rbio, i, sector_nr);
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++ if (!test_bit(index, rbio->csum_bitmap))
+++++++ ++++++++++++++++++ continue;
+++++++ ++++++++++++++++++ if (i == stripe_nr || test_bit(index, rbio->error_bitmap))
+++++++ ++++++++++++++++++ return true;
+++++++ ++++++++++++++++++ }
+++++++ ++++++++++++++++++ return false;
+++++++ ++++++++++++++++++}
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++/*
+++++++ ++++++++++++++++++ * After the read phase of a sub-stripe write, return the sectors it repaired
+++++++ ++++++++++++++++++ * that the write phase should also write back, or NULL if there are none.
+++++++ ++++++++++++++++++ *
+++++++ ++++++++++++++++++ * recover_sectors() leaves the error bit set on every sector it rebuilt: a
+++++++ ++++++++++++++++++ * data sector that failed its csum or its read, every sector of a missing
+++++++ ++++++++++++++++++ * device, and the extra stripe that recover_vertical_retry() found stale.
+++++++ ++++++++++++++++++ * The rebuilt sectors are in the stripe pages and the new P and Q are
+++++++ ++++++++++++++++++ * generated from them, so without a write-back the disk copies stay stale
+++++++ ++++++++++++++++++ * until read repair or scrub finds them.
+++++++ ++++++++++++++++++ *
+++++++ ++++++++++++++++++ * Nothing is written to a missing device. A data sector is written back only
+++++++ ++++++++++++++++++ * if its rebuild is verified (rmw_repair_verified()). P and Q are generated
+++++++ ++++++++++++++++++ * from the data either way, so a failed or stale one is always written back.
+++++++ ++++++++++++++++++ *
+++++++ ++++++++++++++++++ * This rbio has not written anything yet, so none of these is a retry of a
+++++++ ++++++++++++++++++ * write that just failed here.
+++++++ ++++++++++++++++++ */
+++++++ ++++++++++++++++++static unsigned long *rmw_repaired_sectors(struct btrfs_raid_bio *rbio)
+++++++ ++++++++++++++++++{
+++++++ ++++++++++++++++++ unsigned long *repaired = NULL;
+++++++ ++++++++++++++++++ int total_sector_nr;
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++ for_each_set_bit(total_sector_nr, rbio->error_bitmap, rbio->nr_sectors) {
+++++++ ++++++++++++++++++ const int stripe = total_sector_nr / rbio->stripe_nsectors;
+++++++ ++++++++++++++++++ const int sectornr = total_sector_nr % rbio->stripe_nsectors;
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++ if (!rbio->bioc->stripes[stripe].dev->bdev)
+++++++ ++++++++++++++++++ continue;
+++++++ ++++++++++++++++++ if (stripe < rbio->nr_data &&
+++++++ ++++++++++++++++++ !rmw_repair_verified(rbio, stripe, sectornr))
+++++++ ++++++++++++++++++ continue;
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++ if (!repaired) {
+++++++ ++++++++++++++++++ /* Without memory the sectors just stay stale on disk. */
+++++++ ++++++++++++++++++ repaired = bitmap_zalloc(rbio->nr_sectors, GFP_NOFS);
+++++++ ++++++++++++++++++ if (!repaired)
+++++++ ++++++++++++++++++ return NULL;
+++++++ ++++++++++++++++++ }
+++++++ ++++++++++++++++++ set_bit(total_sector_nr, repaired);
+++++++ ++++++++++++++++++ }
+++++++ ++++++++++++++++++ return repaired;
+++++++ ++++++++++++++++++}
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++/*
+++++++ ++++++++++++++++++ * Add write bios for the repaired sectors that the write phase does not write
+++++++ ++++++++++++++++++ * anyway: data sectors no bio of this rbio covers, and P/Q outside the
+++++++ ++++++++++++++++++ * vertical stripes being written. The others are dropped from @repaired.
+++++++ ++++++++++++++++++ * The replace target gets the same sectors as its source.
+++++++ ++++++++++++++++++ */
+++++++ ++++++++++++++++++static int rmw_assemble_repair_bios(struct btrfs_raid_bio *rbio,
+++++++ ++++++++++++++++++ struct bio_list *bio_list,
+++++++ ++++++++++++++++++ unsigned long *repaired)
+++++++ ++++++++++++++++++{
+++++++ ++++++++++++++++++ int total_sector_nr;
+++++++ ++++++++++++++++++ int ret;
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++ for_each_set_bit(total_sector_nr, repaired, rbio->nr_sectors) {
+++++++ ++++++++++++++++++ const int stripe = total_sector_nr / rbio->stripe_nsectors;
+++++++ ++++++++++++++++++ const int sectornr = total_sector_nr % rbio->stripe_nsectors;
+++++++ ++++++++++++++++++ struct sector_ptr *sector;
+++++++ ++++++++++++++++++ bool written;
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++ if (stripe < rbio->nr_data)
+++++++ ++++++++++++++++++ written = sector_in_rbio(rbio, stripe, sectornr, 1);
+++++++ ++++++++++++++++++ else
+++++++ ++++++++++++++++++ written = test_bit(sectornr, &rbio->dbitmap);
+++++++ ++++++++++++++++++ if (written) {
+++++++ ++++++++++++++++++ clear_bit(total_sector_nr, repaired);
+++++++ ++++++++++++++++++ continue;
+++++++ ++++++++++++++++++ }
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++ sector = rbio_stripe_sector(rbio, stripe, sectornr);
+++++++ ++++++++++++++++++ ret = rbio_add_io_sector(rbio, bio_list, sector, stripe,
+++++++ ++++++++++++++++++ sectornr, REQ_OP_WRITE);
+++++++ ++++++++++++++++++ if (ret)
+++++++ ++++++++++++++++++ goto error;
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++ if (likely(!rbio->bioc->replace_nr_stripes) ||
+++++++ ++++++++++++++++++ stripe != rbio->bioc->replace_stripe_src)
+++++++ ++++++++++++++++++ continue;
+++++++ ++++++++++++++++++ ret = rbio_add_io_sector(rbio, bio_list, sector,
+++++++ ++++++++++++++++++ rbio->real_stripes, sectornr,
+++++++ ++++++++++++++++++ REQ_OP_WRITE);
+++++++ ++++++++++++++++++ if (ret)
+++++++ ++++++++++++++++++ goto error;
+++++++ ++++++++++++++++++ }
+++++++ ++++++++++++++++++ return 0;
+++++++ ++++++++++++++++++error:
+++++++ ++++++++++++++++++ bio_list_put(bio_list);
+++++++ ++++++++++++++++++ return -EIO;
+++++++ ++++++++++++++++++}
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++/*
+++++++ ++++++++++++++++++ * A write-back that fails leaves the sector as stale as it was before this
+++++++ ++++++++++++++++++ * write, so it does not count against the tolerance of the write itself.
+++++++ ++++++++++++++++++ */
+++++++ ++++++++++++++++++static void rmw_finish_repair_bios(struct btrfs_raid_bio *rbio,
+++++++ ++++++++++++++++++ unsigned long *repaired)
+++++++ ++++++++++++++++++{
+++++++ ++++++++++++++++++ int written = 0;
+++++++ ++++++++++++++++++ int failed = 0;
+++++++ ++++++++++++++++++ int total_sector_nr;
+++++++ ++++++++++++++++++
+++++++ ++++++++++++++++++ for_each_set_bit(total_sector_nr, repaired, rbio->nr_sectors) {
+++++++ ++++++++++++++++++ written++;
+++++++ ++++++++++++++++++ if (test_and_clear_bit(total_sector_nr, rbio->error_bitmap))
+++++++ ++++++++++++++++++ failed++;
+++++++ ++++++++++++++++++ }
+++++++ ++++++++++++++++++ if (written)
+++++++ ++++++++++++++++++ btrfs_debug_rl(rbio->bioc->fs_info,
+++++++ ++++++++++++++++++ "sub-stripe write for full stripe %llu wrote back %d repaired sectors, %d failed",
+++++++ ++++++++++++++++++ rbio->bioc->full_stripe_logical, written, failed);
+++++++ ++++++++++++++++++}
+++++++ ++++++++++++++++++
+++++++ + + ++ ++ ++++++++static void rmw_rbio(struct btrfs_raid_bio *rbio)
+++++++ + + ++ ++ ++++++++{
+++++++ + + ++ ++ ++++++++ struct bio_list bio_list;
+++++++ + + ++ ++ ++++++++ int sectornr;
+++++++ ++++++++++++++++++ unsigned long *repaired = NULL;
+++++++ + + ++ ++ ++++++++ int ret = 0;
+++++++ + + ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Write-hole invariant check: every write must land in a stripe
+++++++++++ ++ ++ ++++++++ * covered by a live stripe run when stripe-exclusive allocation is
+++++++++++ ++ ++ ++++++++ * enabled. The bios gathered in this rbio have not reported their
+++++++++++ ++ ++ ++++++++ * IO done yet, so their runs cannot drain under us.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ btrfs_stripe_check_write(rbio->bioc->fs_info,
+++++++++++ ++ ++ ++++++++ rbio->bioc->full_stripe_logical,
+++++++++++ ++ ++ ++++++++ !rbio_is_full(rbio));
+++++++++++ ++ ++ ++++++++
+++++++ + + ++ ++ ++++++++ /*
+++++++ + + ++ ++ ++++++++ * Allocate the pages for parity first, as P/Q pages will always be
+++++++ + + ++ ++ ++++++++ * needed for both full-stripe and sub-stripe writes.
+++++++ + + ++ ++ ++++++++ */
+++++++ + + ++ ++ ++++++++ ret = alloc_rbio_parity_pages(rbio);
+++++++ + + ++ ++ ++++++++ if (ret < 0)
+++++++ + + ++ ++ ++++++++ goto out;
+++++++ + + ++ ++ ++++++++
+++++++ + + ++ ++ ++++++++ /*
+++++++ + + ++ ++ ++++++++ * Either full stripe write, or we have every data sector already
+++++++ + + ++ ++ ++++++++ * cached, can go to write path immediately.
+++++++++++ ++ ++ ++++++++ *
+++++++++++ ++ ++ ++++++++ * Try padding before trusting the cache: a cached rbio can hold pages
+++++++++++ ++ ++ ++++++++ * from the stripe's previous life, before its extents were freed and
+++++++++++ ++ ++ ++++++++ * the stripe re-claimed, and parity computed from them matches the
+++++++++++ ++ ++ ++++++++ * disk only if every uncovered column still holds that data. Padding
+++++++++++ ++ ++ ++++++++ * writes every uncovered column, so the stripe needs nothing on disk.
+++++++ + + ++ ++ ++++++++ */
- - if (!rbio_is_full(rbio) && need_read_stripe_sectors(rbio)) {
+++++++++++ ++ ++ ++++++++ if (!rbio_is_full(rbio) && !rmw_try_pad_full(rbio) &&
+++++++++++ ++ ++ ++++++++ need_read_stripe_sectors(rbio)) {
+++++++++++ ++ ++ ++++++++ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+++++++++++ ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /*
+++++++++++ ++ ++ ++++++++ * Every data read-modify-write, parked or not. With
+++++++++++ ++ ++ ++++++++ * allow_rmw empty and a covered block group this is the
+++++++++++ ++ ++ ++++++++ * count that should be zero.
+++++++++++ ++ ++ ++++++++ */
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.data_rmw);
+++++++++++ ++ ++ ++++++++ if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA))
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_reads);
+++++++++++ ++ ++ ++++++++ rbio_count_rmw_reason(rbio);
+++++++++++ ++ ++ ++++++++ /* A parked-then-flushed write that still had to read. */
+++++++++++ ++ ++ ++++++++ if (test_bit(RBIO_WAS_PARKED_BIT, &rbio->flags))
+++++++++++ ++ ++ ++++++++ atomic64_inc(&fs_info->stripe_park_stats.rmw_reads);
+++++++ + + ++ ++ ++++++++ /*
+++++++ + + ++ ++ ++++++++ * Now we're doing sub-stripe write, also need all data stripes
+++++++ + + ++ ++ ++++++++ * to do the full RMW.
+++++++ + + ++ ++ ++++++++ */
+++++++ + + ++ ++ ++++++++ ret = alloc_rbio_data_pages(rbio);
+++++++ + + ++ ++ ++++++++ if (ret < 0)
+++++++ + + ++ ++ ++++++++ goto out;
+++++++ + + ++ ++ ++++++++
+++++++ + + ++ ++ ++++++++ index_rbio_pages(rbio);
+++++++ + + ++ ++ ++++++++
+++++++ + + ++ ++ ++++++++ ret = rmw_read_wait_recover(rbio);
+++++++ + + ++ ++ ++++++++ if (ret < 0)
+++++++ + + ++ ++ ++++++++ goto out;
+++++++ ++++++++++++++++++ repaired = rmw_repaired_sectors(rbio);
+++++++ + + ++ ++ ++++++++ }
+++++++ + + ++ ++ ++++++++
+++++++++++ ++ ++ ++++++++ /* Still a sub-stripe write: this stripe is being modified in place. */
+++++++++++ ++ ++ ++++++++ if (!rbio_is_full(rbio) && !test_bit(RBIO_PADDED_BIT, &rbio->flags))
+++++++++++ ++ ++ ++++++++ report_uncovered_rmw(rbio);
+++++++++++ ++ ++ ++++++++
+++++++ + + ++ ++ ++++++++ /*
+++++++ + + ++ ++ ++++++++ * At this stage we're not allowed to add any new bios to the
+++++++ + + ++ ++ ++++++++ * bio list any more, anyone else that wants to change this stripe
+++++++ + + ++ ++ ++++++++ * needs to do their own rmw.
*/
spin_lock(&rbio->bio_list_lock);
set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
pointers[nr_data] = kmap_local_sector(&p_sector);
for_each_set_bit(sectornr, &rbio->dbitmap, rbio->stripe_nsectors) {
--------- ---------------- struct sector_ptr *sector;
--------- ---------------- void *parity;
--------- ----------------
--------- ---------------- /* first collect one page from each data stripe */
--------- ---------------- for (stripe = 0; stripe < nr_data; stripe++) {
--------- ---------------- sector = sector_in_rbio(rbio, stripe, sectornr, 0);
--------- ---------------- pointers[stripe] = kmap_local_sector(sector);
------- -------------- - }
------- -------------- -
------- -------------- - if (has_qstripe) {
------- -------------- - assert_rbio(rbio);
------- -------------- - /* RAID6, call the library function to fill in our P/Q */
------- -------------- - raid6_call.gen_syndrome(rbio->real_stripes, sectorsize,
------- -------------- - pointers);
------- -------------- - } else {
------- -------------- - /* raid5 */
------- -------------- - memcpy(pointers[nr_data], pointers[0], sectorsize);
------- -------------- - run_xor(pointers + 1, nr_data - 1, sectorsize);
+++++++++ ++++++++++++++++ stripe = verify_one_parity_sector(rbio, pointers, sectornr,
+++++++++ ++++++++++++++++ !test_bit(sectornr, &read_errors));
+++++++++ ++++++++++++++++ if (stripe >= 0) {
+++++++++ ++++++++++++++++ __set_bit(sectornr, &rebuilt);
+++++++++ ++++++++++++++++ rebuilt_stripe[sectornr] = stripe;
}
-- -
-- - if (has_qstripe) {
-- - assert_rbio(rbio);
-- - /* RAID6, call the library function to fill in our P/Q */
-- - raid6_call.gen_syndrome(rbio->real_stripes, sectorsize,
-- - pointers);
-- - } else {
-- - /* raid5 */
-- - memcpy(pointers[nr_data], pointers[0], sectorsize);
-- - run_xor(pointers + 1, nr_data - 1, sectorsize);
-- - }
--------- ----------------
--------- ---------------- /* Check scrubbing parity and repair it */
--------- ---------------- sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr);
--------- ---------------- parity = kmap_local_sector(sector);
--------- ---------------- if (memcmp(parity, pointers[rbio->scrubp], sectorsize) != 0)
--------- ---------------- memcpy(parity, pointers[rbio->scrubp], sectorsize);
--------- ---------------- else
--------- ---------------- /* Parity is right, needn't writeback */
--------- ---------------- bitmap_clear(&rbio->dbitmap, sectornr, 1);
--------- ---------------- kunmap_local(parity);
--------- ----------------
--------- ---------------- for (stripe = nr_data - 1; stripe >= 0; stripe--)
--------- ---------------- kunmap_local(pointers[stripe]);
}
+++++++++ ++++++++++++++++ count_parity_scrub_errors(rbio, uncounted, rebuilt, rebuilt_stripe);
kunmap_local(pointers[nr_data]);
__free_page(phys_to_page(p_sector.paddr));
q_sector.has_paddr = false;
}
- - * nothing back to the scrubbed device. A running dev-replace still
- - * copies to its target.
+++++++++ ++++++++++++++++ /* A read-only scrub only checks: submit no write. */
+++++++++ ++++++++++++++++ if (rbio->scrub_readonly)
+++++++++ ++++++++++++++++ goto submit_write;
+++++++++ ++++++++++++++++
++++++++ ++++++++++++++ + /*
++++++++ ++++++++++++++ + * After a transaction abort or another filesystem error, write
+++++++++ ++++++++++++++++ * nothing back to the scrubbed devices, rebuilt data included. A
+++++++++ ++++++++++++++++ * running dev-replace still copies parity to its target.
++++++++ ++++++++++++++ + */
++++++++ ++++++++++++++ + if (!btrfs_can_submit_repair_write(rbio->bioc->fs_info))
++++++++ ++++++++++++++ + goto replace;
++++++++ ++++++++++++++ +
+++++++++ ++++++++++++++++ /* Write the data sectors rebuilt from P and Q. */
+++++++++ ++++++++++++++++ for_each_set_bit(sectornr, &rebuilt, rbio->stripe_nsectors) {
+++++++++ ++++++++++++++++ const struct btrfs_io_stripe *io_stripe;
+++++++++ ++++++++++++++++
+++++++++ ++++++++++++++++ stripe = rebuilt_stripe[sectornr];
+++++++++ ++++++++++++++++ io_stripe = &bioc->stripes[stripe];
+++++++++ ++++++++++++++++ btrfs_warn_rl(bioc->fs_info,
+++++++++ ++++++++++++++++"scrub: raid6 data at logical %llu on devid %llu physical %llu did not match P and Q, rebuilt it",
+++++++++ ++++++++++++++++ bioc->full_stripe_logical +
+++++++++ ++++++++++++++++ btrfs_stripe_nr_to_offset(stripe) +
+++++++++ ++++++++++++++++ ((u64)sectornr << bioc->fs_info->sectorsize_bits),
+++++++++ ++++++++++++++++ io_stripe->dev->devid,
+++++++++ ++++++++++++++++ io_stripe->physical +
+++++++++ ++++++++++++++++ ((u64)sectornr << bioc->fs_info->sectorsize_bits));
+++++++++ ++++++++++++++++ ret = rbio_add_io_sector(rbio, &bio_list,
+++++++++ ++++++++++++++++ rbio_stripe_sector(rbio, stripe, sectornr),
+++++++++ ++++++++++++++++ stripe, sectornr, REQ_OP_WRITE);
+++++++++ ++++++++++++++++ if (ret)
+++++++++ ++++++++++++++++ goto cleanup;
+++++++++ ++++++++++++++++ }
+++++++++ ++++++++++++++++
/*
* time to start writing. Make bios for everything from the
* higher layers (the bio_list in our rbio) and our p/q. Ignore