u64 owner; /* LOG class: btrfs_ino of the owner */
enum btrfs_stripe_run_class class;
bool open; /* accepting allocations */
+ /*
+ * One bit per sector of the run, set while the sector holds an
+ * allocation someone still intends to write. Padding uses it to find
+ * dead sectors below the frontier (allocations abandoned before they
+ * were written), which are as safe to zero-fill as space above it.
+ */
+ unsigned long *live;
+ u32 live_nbits; /* sectors @live covers; caps growth */
};
static bool open_stripe_run_drained(const struct btrfs_open_stripe_run *run)
return band;
}
+/*
+ * Growth headroom for a run's liveness map, in full stripes.
+ *
+ * The map is sized once and never reallocated, because the only place a run
+ * grows -- the growable frontier in btrfs_alloc_from_open_stripe() -- holds
+ * bg->stripe_run_lock with interrupts disabled, where nothing may sleep.
+ * Sizing it beyond the run's initial span is what lets that path work at
+ * all; a run that would outgrow its map stops growing instead, and the
+ * allocation opens a fresh run.
+ */
+#define BTRFS_STRIPE_RUN_LIVE_STRIPES 32
+
+/*
+ * Allocate a run's liveness map. Must be called before taking
+ * bg->stripe_run_lock: this sleeps. Sets *nbits to what the map covers.
+ */
+static unsigned long *alloc_run_live_map(struct btrfs_block_group *bg,
+ u64 run_len, u32 *nbits)
+{
+ const u64 span = max_t(u64, run_len,
+ (u64)BTRFS_STRIPE_RUN_LIVE_STRIPES *
+ bg->full_stripe_len);
+ unsigned long *map;
+
+ *nbits = span >> bg->fs_info->sectorsize_bits;
+ map = bitmap_zalloc(*nbits, GFP_NOFS);
+ if (!map)
+ *nbits = 0;
+ return map;
+}
+
+/* Mark [start, start+len) live (allocated) or dead (abandoned). */
+static void run_live_set(struct btrfs_open_stripe_run *run, u64 start, u64 len,
+ bool live)
+{
+ const u32 bits = run->bg->fs_info->sectorsize_bits;
+ u32 first, nr;
+
+ if (!run->live || !run->live_nbits || start < run->start)
+ return;
+ first = (start - run->start) >> bits;
+ nr = len >> bits;
+ /*
+ * Growth is capped at the map size, so this should be unreachable. If
+ * it happens, retire the map: btrfs_stripe_run_pad_mask() then declines
+ * the run, and the write falls back to read-modify-write instead of
+ * zero-filling a sector that may be live.
+ */
+ if (unlikely(first + nr > run->live_nbits)) {
+ WARN_ON_ONCE(1);
+ run->live_nbits = 0;
+ return;
+ }
+ if (live)
+ bitmap_set(run->live, first, nr);
+ else
+ bitmap_clear(run->live, first, nr);
+}
+
/* Free a drained run. Caller holds bg->stripe_run_lock. */
static void free_open_stripe_run(struct btrfs_block_group *bg,
struct btrfs_open_stripe_run *run)
{
ASSERT(open_stripe_run_drained(run));
list_del(&run->list);
+ bitmap_free(run->live);
kfree(run);
wake_up_var(&bg->open_stripe_runs);
}
}
if (run) {
*ret_offset = run->offset;
+ run_live_set(run, run->offset, num_bytes, true);
run->offset += num_bytes;
run->inflight_bytes += num_bytes;
bg->stripe_open_remainder -= num_bytes;
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
- new_run = kmalloc(sizeof(*new_run), GFP_NOFS);
+ /* Zeroed, so the error paths can free ->live and ->written. */
+ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
if (!new_run)
return -ENOMEM;
goto out;
}
+ /* Sleeps, so it has to happen before the locks below. */
+ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
+
/*
* Install under fs_info->open_stripe_lock so that the block group
* membership, the retire sequence read and the run installation are
if (!r || r->end != start)
continue;
+ /*
+ * Only grow as far as the run's liveness map reaches; it
+ * cannot be reallocated here (see alloc_run_live_map()).
+ * Beyond that, leave the run alone and open a new one.
+ */
+ if (((start + len - r->start) >> fs_info->sectorsize_bits) >
+ r->live_nbits)
+ continue;
ASSERT(r->open);
bg->open_stripe[class][band] = NULL;
*ret_offset = r->offset;
+ /*
+ * Mark the handed-out bytes live before anything can consult
+ * the map: this allocation begins at the old tail, usually
+ * mid-stripe, and a sector the map calls dead is a sector a
+ * neighbour's padded write may zero-fill.
+ */
+ run_live_set(r, r->offset, num_bytes, true);
r->end = start + len;
r->offset += num_bytes;
r->inflight_bytes += num_bytes;
new_run->inflight_bytes = num_bytes;
new_run->open_seq = fs_info->stripe_retire_seq;
new_run->open = (new_run->offset != new_run->end);
+ run_live_set(new_run, start, num_bytes, true);
list_add_tail(&new_run->list, &bg->open_stripe_runs);
bg->stripe_open_remainder += new_run->end - new_run->offset;
/*
new_run = NULL;
ret = 0;
out:
+ /* Unused on every path that did not install it, including errors. */
+ if (new_run)
+ bitmap_free(new_run->live);
kfree(new_run);
return ret;
}
continue;
if (num_bytes <= run->end - run->offset) {
*ret_offset = run->offset;
+ run_live_set(run, run->offset, num_bytes, true);
run->offset += num_bytes;
run->inflight_bytes += num_bytes;
bg->stripe_open_remainder -= num_bytes;
stripe_open_remainder_sync(bg);
}
- new_run = kmalloc(sizeof(*new_run), GFP_NOFS);
+ /* Zeroed, so the error paths can free ->live and ->written. */
+ new_run = kzalloc(sizeof(*new_run), GFP_NOFS);
if (!new_run)
return -ENOMEM;
goto out;
}
+ /* Sleeps, so it has to happen before the locks below. */
+ new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
+
/* See the installation comment in btrfs_alloc_from_open_stripe(). */
spin_lock(&fs_info->open_stripe_lock);
if (list_empty(&bg->open_stripe_bg_list)) {
new_run->inflight_bytes = num_bytes;
new_run->open_seq = fs_info->stripe_retire_seq;
new_run->open = (new_run->offset != new_run->end);
+ run_live_set(new_run, start, num_bytes, true);
list_add_tail(&new_run->list, &bg->open_stripe_runs);
bg->stripe_open_remainder += new_run->end - new_run->offset;
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
new_run = NULL;
ret = 0;
out:
+ /* Unused on every path that did not install it, including errors. */
+ if (new_run)
+ bitmap_free(new_run->live);
kfree(new_run);
return ret;
}
}
/*
- * For the raid56 pad-to-full optimization: a sub-stripe write into a
- * stripe covered by a live stripe run may zero-fill (instead of read) any
- * sector at or past the run's allocation frontier, because such sectors
- * have never been written in the run's lifetime and, once the run closes,
- * its stripe's tail cannot be reallocated while the stripe holds live
- * data. Returns true and sets *pad_from to the frontier when the stripe
- * at [stripe_start, stripe_start + stripe_len) has a paddable tail. The
- * frontier can grow right after this returns; that is safe, because the
- * later allocation's write serializes behind the caller's stripe lock
- * and overwrites the zeros.
+ * Lower an allocation ceiling to the end of the last live sector at or above
+ * @floor. Sectors between it and the frontier were allocated and abandoned:
+ * they never get a bio, so a parked rbio counting on them would leave only
+ * when kicked or at its park deadline. Caller holds bg->stripe_run_lock.
*/
+static u64 run_live_ceiling(const struct btrfs_open_stripe_run *run, u64 floor,
+ u64 ceiling)
+{
+ const u32 bits = run->bg->fs_info->sectorsize_bits;
+ u32 first;
+ u32 i;
+
+ if (!run->live || !run->live_nbits || floor < run->start ||
+ ceiling <= floor)
+ return ceiling;
+ first = (floor - run->start) >> bits;
+ i = (ceiling - run->start) >> bits;
+ if (i > run->live_nbits)
+ return ceiling;
+ while (i > first && !test_bit(i - 1, run->live))
+ i--;
+ return run->start + ((u64)i << bits);
+}
+
/*
* How far has the covering run allocated into this stripe? Returns the
* exclusive end of the allocated prefix, clamped to the stripe, or U64_MAX
continue;
ceiling = clamp(run->offset, stripe_start,
stripe_start + stripe_len);
+ ceiling = run_live_ceiling(run, stripe_start, ceiling);
break;
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
return ceiling;
}
-bool btrfs_stripe_run_pad_start(struct btrfs_fs_info *fs_info,
- u64 stripe_start, u64 stripe_len,
- u64 *pad_from,
- struct btrfs_stripe_pad_info *info)
+/*
+ * Which sectors of this stripe may be padded?
+ *
+ * Fills @dead with one bit per sector of the stripe, set where the sector
+ * holds nothing anyone intends to write: space the run has not handed out
+ * yet, and space it handed out but got back before the write was issued.
+ * A caller may zero-fill exactly those and write the whole stripe.
+ *
+ * Returns false if no run covers the stripe, in which case nothing may be
+ * padded: without a run there is no claim of exclusivity, and an uncovered
+ * sector may hold committed data.
+ *
+ * This replaces asking where the run's frontier is. The frontier answers
+ * "what has never been allocated", which misses allocations that were
+ * abandoned below it -- a tree block COWed twice in one transaction, a
+ * reservation released, an allocation the finder discarded. Those leave
+ * holes that are equally safe to pad, and skipping them is the difference
+ * between padding most partial stripes and padding all of them.
+ *
+ * @info, when non-NULL, reports the covering run's state at the moment of
+ * the decision, for the pad-decline trace: a later extent-tree dump cannot
+ * reconstruct where the frontier was when the write arrived.
+ */
+bool btrfs_stripe_run_pad_mask(struct btrfs_fs_info *fs_info, u64 stripe_start,
+ u64 stripe_len, unsigned long *dead,
+ struct btrfs_stripe_pad_info *info)
{
+ const u32 bits = fs_info->sectorsize_bits;
+ const u32 nsectors = stripe_len >> bits;
struct btrfs_block_group *bg;
struct btrfs_open_stripe_run *run;
- atomic64_t *decline = NULL;
unsigned long flags;
bool ret = false;
spin_lock_irqsave(&bg->stripe_run_lock, flags);
list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ u32 first;
+
if (stripe_start < run->start || stripe_start >= run->end)
continue;
if (info) {
info->run_end = run->end;
info->run_offset = run->offset;
info->inflight_bytes = run->inflight_bytes;
- info->reason = BTRFS_STRIPE_PAD_OK;
- }
- if (run->offset > stripe_start &&
- run->offset < stripe_start + stripe_len) {
- *pad_from = run->offset;
- ret = true;
- } else if (run->offset >= stripe_start + stripe_len) {
- /*
- * The frontier is past this stripe: every sector of
- * it is allocated, so there is nothing dead to pad
- * and the write is waiting on a neighbour's data.
- */
- decline = &fs_info->stripe_park_stats.pad_decline_live;
- if (info)
- info->reason = BTRFS_STRIPE_PAD_LIVE;
- } else {
- /* The frontier has not reached the stripe at all. */
- decline = &fs_info->stripe_park_stats.pad_decline_unallocated;
- if (info)
- info->reason = BTRFS_STRIPE_PAD_UNALLOCATED;
}
+ if (!run->live || !run->live_nbits)
+ break;
+ /*
+ * The stripe must lie inside the run's span. A closed run
+ * has shrunk its end to the frontier, so its final stripe
+ * overhangs: the sectors past end are the returned tail,
+ * sub-stripe free space nothing can claim while the stripe
+ * holds live data. They were never allocated, their live
+ * bits are clear, and they are exactly what a kicked park
+ * needs to pad. An open run's end is stripe aligned, so
+ * overhang there means a stripe the run does not own.
+ */
+ if (stripe_start + stripe_len > run->end && run->open)
+ break;
+
+ first = (stripe_start - run->start) >> bits;
+ /*
+ * And wholly inside the map. Declining here costs a
+ * read-modify-write; reading past the map would be a lie about
+ * which sectors are dead, and padding acts on that by
+ * overwriting them.
+ */
+ if (first + nsectors > run->live_nbits)
+ break;
+ bitmap_zero(dead, nsectors);
+ for (u32 i = 0; i < nsectors; i++)
+ if (!test_bit(first + i, run->live))
+ set_bit(i, dead);
+ if (info)
+ info->reason = BTRFS_STRIPE_PAD_OK;
+ ret = true;
break;
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
if (!ret)
- atomic64_inc(decline ? decline :
- &fs_info->stripe_park_stats.pad_decline_no_run);
+ atomic64_inc(&fs_info->stripe_park_stats.pad_decline_no_run);
btrfs_put_block_group(bg);
return ret;
}
+
/*
* Can stripe-exclusive allocation be enabled on this filesystem? Called for
* the "stripe_alloc" filesystem property, on a mounted (or mounting, options
return ret;
}
+/*
+ * Report bytes that will never be written: an allocation released before
+ * its write was issued. Same accounting as a completion, plus the sectors
+ * go dead so a later partial write to their stripe can pad over them
+ * instead of falling back to read-modify-write.
+ */
+void btrfs_open_stripe_write_abandoned(struct btrfs_block_group *bg, u64 start,
+ u64 num_bytes)
+{
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (start >= run->start && start < run->offset) {
+ ASSERT(start + num_bytes <= run->offset);
+ ASSERT(run->inflight_bytes >= num_bytes);
+ run_live_set(run, start, num_bytes, false);
+ run->inflight_bytes -= num_bytes;
+ if (open_stripe_run_drained(run))
+ free_open_stripe_run(bg, run);
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+}
+
+/*
+ * Report completed IO for an allocation from any stripe run. Every
+ * allocated byte must be reported exactly once, here or through
+ * btrfs_open_stripe_write_abandoned(), in any number of pieces (e.g. after
+ * ordered extent splits).
+ */
void btrfs_open_stripe_write_done(struct btrfs_block_group *bg, u64 start,
u64 num_bytes)
{
}
/*
- * Entry-blind complement to the by-size fast path in
- * btrfs_claim_free_stripe_run(): walk the aligned stripe grid in
- * address order, accumulating contiguous free coverage
+ * Find a fully free, stripe-aligned run of up to @want_bytes.
+ *
+ * Returns the free space entry containing it with @run_start and @run_len
+ * describing the run, or NULL if there is none. The caller decides what to
+ * do with it: claiming it for a stripe run, or handing it to a cluster.
+ *
+ * Walks the by-size index (largest max contiguous free run first) rather
+ * than the by-offset index. An entry whose largest contiguous free run is
+ * smaller than a full stripe cannot contain a fully-free stripe, and every
+ * following entry is no larger, so we stop at the first such entry.
+ * Near-full, where the free space degenerates into many sub-stripe holes,
+ * this is an O(1) fast fail instead of a scan of the whole free space tree
+ * on every allocation.
+ *
+ * ctl->tree_lock must be held.
+ */
+static struct btrfs_free_space *find_free_stripe_run(
+ struct btrfs_block_group *block_group,
+ u64 want_bytes, u64 *run_start,
+ u64 *run_len)
+{
+ struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
+ const u64 fsl = block_group->full_stripe_len;
+ struct btrfs_free_space *entry;
+ struct rb_node *node;
+
+ lockdep_assert_held(&ctl->tree_lock);
+
+ if (ctl->free_space < fsl)
+ return NULL;
+
+ for (node = rb_first_cached(&ctl->free_space_bytes); node;
+ node = rb_next(node)) {
+ entry = rb_entry(node, struct btrfs_free_space, bytes_index);
+
+ if (get_max_extent_size(entry) < fsl)
+ break;
+
+ if (entry->bitmap) {
+ if (find_stripe_run_in_bitmap(ctl, entry, want_bytes,
+ run_start, run_len) &&
+ btrfs_stripe_run_range_usable(block_group,
+ *run_start, run_len))
+ return entry;
+ continue;
+ }
+
+ *run_start = stripe_run_align(block_group, entry->offset);
+ if (*run_start + fsl > entry->offset + entry->bytes)
+ continue;
+ *run_len = min(want_bytes,
+ div64_u64(entry->offset + entry->bytes - *run_start,
+ fsl) * fsl);
+ if (!btrfs_stripe_run_range_usable(block_group, *run_start,
+ run_len))
+ continue;
+ return entry;
+ }
+ return NULL;
+}
+
+/*
+ * Entry-blind complement to find_free_stripe_run(): walk the aligned
+ * stripe grid in address order, accumulating contiguous free coverage
* with point lookups, looking for a run of fully-free full stripes no
* single entry contains.
*
&block_group->fs_info->discard_ctl;
const u64 fsl = block_group->full_stripe_len;
struct btrfs_free_space *entry;
- struct rb_node *node;
struct btrfs_free_space *spare = NULL;
enum btrfs_trim_state head_trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
bool spare_used = false;
return -ENOSPC;
}
- if (ctl->free_space < fsl)
- goto out;
-
- /*
- * Walk the by-size index (largest max contiguous free run first)
- * rather than the by-offset index. An entry whose largest contiguous
- * free run is smaller than a full stripe cannot contain a fully-free
- * stripe, and every following entry is no larger, so we stop at the
- * first such entry. Near-full, where the free space degenerates into
- * many sub-stripe holes, this is an O(1) fast fail instead of a scan
- * of the whole free space tree on every allocation.
- */
- for (node = rb_first_cached(&ctl->free_space_bytes); node;
- node = rb_next(node)) {
- entry = rb_entry(node, struct btrfs_free_space, bytes_index);
-
- if (get_max_extent_size(entry) < fsl)
- break;
-
- if (entry->bitmap) {
- if (find_stripe_run_in_bitmap(ctl, entry, want_bytes,
- &run_start, &run_len) &&
- btrfs_stripe_run_range_usable(block_group,
- run_start, &run_len))
- goto found;
- continue;
+ entry = find_free_stripe_run(block_group, want_bytes, &run_start, &run_len);
+ if (!entry) {
+ /*
+ * No single entry holds a full stripe; look for one whose
+ * free space spans entry boundaries. The spare entry is
+ * for the belt-and-braces case of a run strictly inside
+ * one extent entry, whose removal splits it in two.
+ */
+ if (!spare) {
+ spin_unlock(&ctl->tree_lock);
+ spare = kmem_cache_zalloc(btrfs_free_space_cachep,
+ GFP_NOFS);
+ if (!spare)
+ return -ENOMEM;
+ spin_lock(&ctl->tree_lock);
}
-
- run_start = stripe_run_align(block_group, entry->offset);
- if (run_start + fsl > entry->offset + entry->bytes)
- continue;
- run_len = min(want_bytes,
- div64_u64(entry->offset + entry->bytes - run_start,
- fsl) * fsl);
- if (!btrfs_stripe_run_range_usable(block_group, run_start,
- &run_len))
- continue;
- goto found;
- }
-
- /*
- * No single entry holds a full stripe; look for one whose
- * free space spans entry boundaries. The spare entry is
- * for the belt-and-braces case of a run strictly inside
- * one extent entry, whose removal splits it in two.
- */
- spin_unlock(&ctl->tree_lock);
- spare = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS);
- if (!spare)
- return -ENOMEM;
- spin_lock(&ctl->tree_lock);
- if (!find_free_stripe_run_slow(block_group, want_bytes,
- &run_start, &run_len))
+ if (!find_free_stripe_run_slow(block_group, want_bytes,
+ &run_start, &run_len)) {
+ /*
+ * The exhaustive scan found no claimable run at all
+ * (the slow path accepts any run of at least one full
+ * stripe). Cache that verdict so subsequent claims
+ * fail in O(1) until free space is added.
+ */
+ if (btrfs_block_group_done(block_group))
+ block_group->max_claimable_run = 0;
+ goto out;
+ }
+ spare_used = claim_stripe_run_pieces(block_group, run_start,
+ run_len, spare);
+ *start = run_start;
+ *len = run_len;
+ ret = 0;
goto out;
- spare_used = claim_stripe_run_pieces(block_group, run_start,
- run_len, spare);
- *start = run_start;
- *len = run_len;
- ret = 0;
- goto out;
+ }
-found:
if (!btrfs_free_space_trimmed(entry))
atomic64_add(run_len, &discard_ctl->discard_bytes_saved);
return ret;
}
+
/*
* given a cluster, put all of its extents back into the free space
* cache. If a block group is passed, this function will only free
rbio->operation = BTRFS_RBIO_WRITE;
rbio_add_bio(rbio, bio);
+
/*
* Don't plug on full rbios, just get them out the door
* as quickly as we can
}
/*
- * Try to avoid the RMW read phase of a sub-stripe write by zero-filling
- * the uncovered data sectors. Legal only when every uncovered sector
- * lies at or past its stripe run's allocation frontier -- never written
- * in the run's lifetime, and unreachable by other allocations while this
- * stripe holds live data (see btrfs_stripe_run_pad_start()). A frontier
- * that grows under us is safe: the newer allocation's write serializes
- * behind this rbio's stripe lock and lands over the zeros.
+ * Try to avoid the RMW read phase of a sub-stripe write by zero-filling the
+ * uncovered data sectors. Legal only where the stripe's run says a sector
+ * is dead: never allocated, handed back, or allocated with its write still
+ * to come. That write, like any allocation the run makes after this check,
+ * serializes behind this rbio's stripe lock and lands over the zeros.
*/
static bool rmw_try_pad_full(struct btrfs_raid_bio *rbio)
{
struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
const u64 stripe_start = rbio->bioc->full_stripe_logical;
- const u32 sectorsize = fs_info->sectorsize;
const u64 stripe_len = (u64)rbio->nr_data * BTRFS_STRIPE_LEN;
+ const u32 sectorsize = fs_info->sectorsize;
+ const u32 nsectors = rbio->nr_data * rbio->stripe_nsectors;
struct btrfs_stripe_pad_info info;
- const u32 total = rbio->nr_data * rbio->stripe_nsectors;
- u64 pad_from = 0;
+ unsigned long *dead;
u32 covered = 0;
+ bool ok = false;
int i;
if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
return false;
- if (!btrfs_stripe_run_pad_start(fs_info, stripe_start, stripe_len,
- &pad_from, &info)) {
+
+ dead = bitmap_zalloc(nsectors, GFP_NOFS);
+ if (!dead)
+ return false;
+ if (!btrfs_stripe_run_pad_mask(fs_info, stripe_start, stripe_len, dead,
+ &info)) {
trace_btrfs_stripe_pad_decline(fs_info, stripe_start,
- stripe_len, pad_from, 0, total,
+ stripe_len, 0, 0, nsectors,
info.run_start, info.run_end,
info.run_offset,
info.inflight_bytes,
info.reason);
- return false;
+ goto out;
}
/* Populate bio_paddrs so covered sectors are visible below. */
index_rbio_pages(rbio);
+ for (i = 0; i < nsectors; i++)
+ if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
+ covered++;
+
/*
- * Every data sector this rbio does not cover must be paddable. A
- * sector below the frontier is allocated to someone else and this
+ * Every data sector this rbio does not cover must be dead. A live
+ * uncovered sector belongs to another writer's allocation and this
* write does not carry its data, so padding would destroy it: that
* is a distinct refusal from the oracle's, and it is the one that
* says another writer's allocation shares this stripe.
*/
- for (i = 0; i < total; i++)
- if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
- covered++;
-
- for (i = 0; i < total; i++) {
- const int stripe_nr = i / rbio->stripe_nsectors;
- const int sectornr = i % rbio->stripe_nsectors;
- u64 logical = stripe_start + stripe_nr * BTRFS_STRIPE_LEN +
- (u64)sectornr * sectorsize;
-
+ for (i = 0; i < nsectors; i++) {
if (rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR)
continue;
- if (logical < pad_from) {
+ if (!test_bit(i, dead)) {
atomic64_inc(&fs_info->stripe_park_stats.pad_decline_uncovered);
info.reason = BTRFS_STRIPE_PAD_UNCOVERED;
trace_btrfs_stripe_pad_decline(fs_info, stripe_start,
- stripe_len, pad_from,
- covered, total,
+ stripe_len, 0, covered,
+ nsectors,
info.run_start,
info.run_end,
info.run_offset,
info.inflight_bytes,
info.reason);
- return false;
+ goto out;
}
}
+ ok = true;
+out:
+ bitmap_free(dead);
+ if (!ok)
+ return false;
if (alloc_rbio_data_pages(rbio) < 0)
return false;