return !run->open && run->inflight_bytes == 0;
}
+/*
+ * Band for a run with @remaining bytes of unallocated space: the power-of-two
+ * bucket of its remaining sectors (band k == remaining in [2^k, 2^(k+1))
+ * sectors). Runs are kept at most one per band so an allocation can take the
+ * smallest band that fits.
+ */
+static int stripe_run_band(const struct btrfs_block_group *bg, u64 remaining)
+{
+ u32 sectors = remaining >> bg->fs_info->sectorsize_bits;
+ int band;
+
+ if (!sectors)
+ return 0;
+ band = fls(sectors) - 1;
+ if (band >= BTRFS_STRIPE_RUN_NR_BANDS)
+ band = BTRFS_STRIPE_RUN_NR_BANDS - 1;
+ return band;
+}
+
/* Free a drained run. Caller holds bg->stripe_run_lock. */
static void free_open_stripe_run(struct btrfs_block_group *bg,
struct btrfs_open_stripe_run *run)
{
ASSERT(open_stripe_run_drained(run));
- ASSERT(bg->open_stripe[run->class] != run);
list_del(&run->list);
kfree(run);
wake_up_var(&bg->open_stripe_runs);
/*
* Close a run: no further allocation may join it, and its unallocated tail
- * must be returned to the free space cache by the caller (after dropping
- * the lock). Caller holds bg->stripe_run_lock. Returns the tail length
- * and sets *tail_start.
+ * must be returned to the free space cache by the caller (after dropping the
+ * lock). The caller has already removed @run from bg->open_stripe[][].
+ * Caller holds bg->stripe_run_lock. Returns the tail length, sets *tail_start.
*/
static u64 close_open_stripe_run(struct btrfs_block_group *bg,
struct btrfs_open_stripe_run *run,
u64 tail_len;
ASSERT(run->open);
- ASSERT(bg->open_stripe[run->class] == run);
run->open = false;
- bg->open_stripe[run->class] = NULL;
*tail_start = run->offset;
tail_len = run->end - run->offset;
run->end = run->offset;
return tail_len;
}
+/*
+ * Install @run (not currently in any band slot) into its remaining-space band.
+ * If a run already occupies that band, keep whichever has more remaining -- it
+ * can serve larger future allocations, and closing the other traps less -- and
+ * close the loser, returning its tail via *tail_start. Caller holds
+ * bg->stripe_run_lock. Returns the closed tail length, or 0 if no collision.
+ */
+static u64 stripe_run_place(struct btrfs_block_group *bg,
+ struct btrfs_open_stripe_run *run, u64 *tail_start)
+{
+ int class = run->class;
+ int band = stripe_run_band(bg, run->end - run->offset);
+ struct btrfs_open_stripe_run *occ = bg->open_stripe[class][band];
+
+ if (occ) {
+ struct btrfs_open_stripe_run *keep, *shut;
+
+ if (run->end - run->offset >= occ->end - occ->offset) {
+ keep = run;
+ shut = occ;
+ } else {
+ keep = occ;
+ shut = run;
+ }
+ bg->open_stripe[class][band] = keep;
+ return close_open_stripe_run(bg, shut, tail_start);
+ }
+ bg->open_stripe[class][band] = run;
+ return 0;
+}
+
/*
* Allocate num_bytes from the block group's open stripe run, opening a new
* run via btrfs_claim_free_stripe_run() when there is none or the current
u64 tail_len = 0;
u64 start;
u64 len;
+ int band;
int ret;
ASSERT(num_bytes);
*available = 0;
spin_lock_irqsave(&bg->stripe_run_lock, flags);
- run = bg->open_stripe[class];
- if (run && num_bytes <= run->end - run->offset) {
+ /*
+ * Best fit: take the run in the smallest band that can hold this
+ * allocation, and track the largest remaining across all bands as the
+ * retry hint should nothing fit.
+ */
+ run = NULL;
+ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) {
+ struct btrfs_open_stripe_run *r = bg->open_stripe[class][band];
+ u64 rem;
+
+ if (!r)
+ continue;
+ rem = r->end - r->offset;
+ if (rem > *available)
+ *available = rem;
+ if (!run && num_bytes <= rem) {
+ run = r;
+ bg->open_stripe[class][band] = NULL;
+ }
+ }
+ if (run) {
*ret_offset = run->offset;
run->offset += num_bytes;
run->inflight_bytes += num_bytes;
- if (run->offset == run->end) {
- run->open = false;
- bg->open_stripe[class] = NULL;
- }
+ if (run->offset == run->end)
+ run->open = false; /* full; drains then is freed */
+ else
+ tail_len = stripe_run_place(bg, run, &tail_start);
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ if (tail_len)
+ btrfs_add_free_space(bg, tail_start, tail_len);
return 0;
}
- if (run)
- *available = run->end - run->offset;
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
new_run = kmalloc(sizeof(*new_run), GFP_NOFS);
ret = -ENOSPC;
goto out;
}
- /*
- * Close this class's open run if one (re)appeared: it was too small
- * for us, or a racing opener's. Closing is always safe; the last
- * opener wins.
- */
- run = bg->open_stripe[class];
- if (run)
- tail_len = close_open_stripe_run(bg, run, &tail_start);
new_run->bg = bg;
new_run->class = class;
new_run->start = start;
new_run->open_seq = fs_info->stripe_retire_seq;
new_run->open = (new_run->offset != new_run->end);
list_add_tail(&new_run->list, &bg->open_stripe_runs);
- bg->open_stripe[class] = new_run->open ? new_run : NULL;
+ /*
+ * Place the new run in its remaining-space band, closing whichever run
+ * loses a band collision (the last, fuller opener wins). Other bands'
+ * open runs are untouched.
+ */
+ if (new_run->open)
+ tail_len = stripe_run_place(bg, new_run, &tail_start);
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
spin_unlock(&fs_info->open_stripe_lock);
/*
* Close the block group's open runs that predate @seq and return their
- * unallocated tails to the free space cache. Does not wait for inflight
- * IO. Only the bg->open_stripe[] slots can ever be open, so one pass over
- * the classes suffices.
+ * unallocated tails to the free space cache. Does not wait for inflight IO.
+ * Only the bg->open_stripe[][] slots can ever be open, so one pass over the
+ * classes and their remaining-space bands suffices.
*/
static void close_block_group_stripe_runs(struct btrfs_block_group *bg,
u64 seq)
{
- u64 tail_start[BTRFS_STRIPE_RUN_NR_CLASSES];
- u64 tail_len[BTRFS_STRIPE_RUN_NR_CLASSES];
+ u64 tail_start[BTRFS_STRIPE_RUN_NR_CLASSES][BTRFS_STRIPE_RUN_NR_BANDS];
+ u64 tail_len[BTRFS_STRIPE_RUN_NR_CLASSES][BTRFS_STRIPE_RUN_NR_BANDS];
struct btrfs_open_stripe_run *run;
unsigned long flags;
- int class;
+ int class, band;
spin_lock_irqsave(&bg->stripe_run_lock, flags);
for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++) {
- tail_len[class] = 0;
- run = bg->open_stripe[class];
- if (run && run->open_seq < seq)
- tail_len[class] = close_open_stripe_run(bg, run,
- &tail_start[class]);
+ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) {
+ tail_len[class][band] = 0;
+ run = bg->open_stripe[class][band];
+ if (run && run->open_seq < seq) {
+ bg->open_stripe[class][band] = NULL;
+ tail_len[class][band] = close_open_stripe_run(
+ bg, run, &tail_start[class][band]);
+ }
+ }
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
- for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++) {
- if (tail_len[class])
- btrfs_add_free_space(bg, tail_start[class],
- tail_len[class]);
- }
+ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
+ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+ if (tail_len[class][band])
+ btrfs_add_free_space(bg, tail_start[class][band],
+ tail_len[class][band]);
}
/*
BTRFS_STRIPE_RUN_RELOC,
BTRFS_STRIPE_RUN_NR_CLASSES,
};
+
+/*
+ * Open stripe runs are kept in power-of-two bands of remaining free space:
+ * band k holds a run whose remaining is in [2^k, 2^(k+1)) sectors. An
+ * allocation takes the smallest band that fits (segregated best fit), so it
+ * lands in an already-open stripe sized to it instead of forcing a fresh
+ * claim and trapping the old run's tail. 16 bands cover a full stripe of up
+ * to 2^16 sectors, far more than any raid56 geometry.
+ */
+#define BTRFS_STRIPE_RUN_NR_BANDS 16
enum btrfs_block_group_size_class {
/* Unset */
BTRFS_BG_SZ_NONE,
*/
spinlock_t stripe_run_lock;
struct list_head open_stripe_runs;
- struct btrfs_open_stripe_run *open_stripe[BTRFS_STRIPE_RUN_NR_CLASSES];
+ struct btrfs_open_stripe_run
+ *open_stripe[BTRFS_STRIPE_RUN_NR_CLASSES][BTRFS_STRIPE_RUN_NR_BANDS];
/*
* Membership in fs_info->open_stripe_bgs, protected by
* fs_info->open_stripe_lock; holds a block group reference.
goto out;
}
- /* Doesn't fit the remainder: closes the run, opens the next stripe. */
+ /*
+ * A full-stripe allocation cannot fit the open run's remainder, so it
+ * opens a fresh stripe. With multiple cursors the old run stays open in
+ * its remaining-space band (its tail is not trapped); a single cursor
+ * would have closed it here.
+ */
ret = btrfs_alloc_from_open_stripe(cache, fsl, BTRFS_STRIPE_RUN_COW,
&offset, &avail);
if (ret || offset != 2 * fsl) {
ret = -EINVAL;
goto out;
}
- /* The closed run's tail went back to the free space cache... */
- if (!test_check_exists(cache, fsl + 4 * sectorsize,
- fsl - 4 * sectorsize)) {
- test_err("closed run tail missing from free space");
+ /* The old run is still open, so its tail is NOT back in the cache. */
+ if (test_check_exists(cache, fsl + 4 * sectorsize,
+ fsl - 4 * sectorsize)) {
+ test_err("kept-open run tail wrongly freed");
ret = -EINVAL;
goto out;
}
- /* ...but is part of a partial stripe, so the next alloc skips it. */
+ /* A small allocation backtracks to fill the still-open run. */
ret = btrfs_alloc_from_open_stripe(cache, sectorsize,
BTRFS_STRIPE_RUN_COW, &offset,
&avail);
- if (ret || offset != 3 * fsl) {
- test_err("post-close alloc wrong: ret %d offset %llu",
+ if (ret || offset != fsl + 4 * sectorsize) {
+ test_err("backfill alloc wrong: ret %d offset %llu",
ret, offset);
ret = -EINVAL;
goto out;
}
- /* A different class never joins the cow run: fresh stripe. */
+ /*
+ * A different class never joins the cow run: it opens a fresh stripe.
+ * Only stripes #1 and #2 have been claimed so far (multi-cursor kept
+ * #1 open and backfilled it rather than claiming a new stripe for the
+ * small allocations), so the reloc class claims #3.
+ */
ret = btrfs_alloc_from_open_stripe(cache, sectorsize,
BTRFS_STRIPE_RUN_RELOC, &offset,
&avail);
- if (ret || offset != 4 * fsl) {
+ if (ret || offset != 3 * fsl) {
test_err("reloc class alloc wrong: ret %d offset %llu",
ret, offset);
ret = -EINVAL;
ret = btrfs_alloc_from_open_stripe(cache, sectorsize,
BTRFS_STRIPE_RUN_COW, &offset,
&avail);
- if (ret || offset != 3 * fsl + sectorsize) {
+ if (ret || offset != fsl + 5 * sectorsize) {
test_err("cow run lost by class open: ret %d offset %llu",
ret, offset);
ret = -EINVAL;
goto out;
}
- /* Report all data IO complete, in pieces. */
+ /* Report all data IO complete, in pieces, at the offsets allocated. */
btrfs_open_stripe_write_done(cache, fsl, 2 * sectorsize);
btrfs_open_stripe_write_done(cache, fsl + 2 * sectorsize,
2 * sectorsize);
+ btrfs_open_stripe_write_done(cache, fsl + 4 * sectorsize, sectorsize);
+ btrfs_open_stripe_write_done(cache, fsl + 5 * sectorsize, sectorsize);
btrfs_open_stripe_write_done(cache, 2 * fsl, fsl);
btrfs_open_stripe_write_done(cache, 3 * fsl, sectorsize);
- btrfs_open_stripe_write_done(cache, 3 * fsl + sectorsize, sectorsize);
- btrfs_open_stripe_write_done(cache, 4 * fsl, sectorsize);
/*
- * Commit-style retirement: closes both classes' open runs, returns
- * their tails, drains (instantly, all IO reported done) and drops
- * the block group's membership in fs_info->open_stripe_bgs.
+ * Commit-style retirement: closes every open run in every band of both
+ * classes, returns their tails, drains (instantly, all IO reported
+ * done) and drops the block group's membership in
+ * fs_info->open_stripe_bgs. The cow run filled [fsl, fsl+6*sectorsize)
+ * and the reloc run filled [3*fsl, 3*fsl+sectorsize); each tail returns.
*/
btrfs_retire_open_stripes(fs_info, NULL);
- if (!test_check_exists(cache, 3 * fsl + 2 * sectorsize,
- fsl - 2 * sectorsize)) {
+ if (!test_check_exists(cache, fsl + 6 * sectorsize,
+ fsl - 6 * sectorsize)) {
test_err("retired cow run tail missing from free space");
ret = -EINVAL;
goto out;
}
- if (!test_check_exists(cache, 4 * fsl + sectorsize,
+ if (!test_check_exists(cache, 3 * fsl + sectorsize,
fsl - sectorsize)) {
test_err("retired reloc run tail missing from free space");
ret = -EINVAL;
goto out;
}
- /* Stripe #5 was never claimed and is still fully free. */
+ /* Stripe #4 was never claimed and is still fully free. */
ret = btrfs_alloc_from_open_stripe(cache, sectorsize,
BTRFS_STRIPE_RUN_COW, &offset,
&avail);
- if (ret || offset != 5 * fsl) {
+ if (ret || offset != 4 * fsl) {
test_err("post-retire alloc wrong: ret %d offset %llu",
ret, offset);
ret = -EINVAL;
goto out;
}
+ btrfs_open_stripe_write_done(cache, 4 * fsl, sectorsize);
+ btrfs_retire_open_stripes(fs_info, NULL);
+
+ /* Stripe #5 is the last fully-free stripe. */
+ ret = btrfs_alloc_from_open_stripe(cache, sectorsize,
+ BTRFS_STRIPE_RUN_COW, &offset,
+ &avail);
+ if (ret || offset != 5 * fsl) {
+ test_err("last-stripe alloc wrong: ret %d offset %llu",
+ ret, offset);
+ ret = -EINVAL;
+ goto out;
+ }
btrfs_open_stripe_write_done(cache, 5 * fsl, sectorsize);
btrfs_retire_open_stripes(fs_info, NULL);
goto out;
}
- /* Clean up the partial-stripe remainders. */
+ /*
+ * Clean up the partial-stripe remainders: stripe #0's head, stripe #1's
+ * cow tail (10 sectors), and the 15-sector tails of stripes #3 (reloc),
+ * #4 and #5. Stripe #2 was filled completely and has no remainder.
+ */
ret = btrfs_remove_free_space(cache, sectorsize, fsl - sectorsize);
if (ret) {
test_err("error cleaning up head %d", ret);
goto out;
}
- ret = btrfs_remove_free_space(cache, fsl + 4 * sectorsize,
- fsl - 4 * sectorsize);
+ ret = btrfs_remove_free_space(cache, fsl + 6 * sectorsize,
+ fsl - 6 * sectorsize);
if (ret) {
test_err("error cleaning up tail 1 %d", ret);
goto out;
}
- ret = btrfs_remove_free_space(cache, 3 * fsl + 2 * sectorsize,
- fsl - 2 * sectorsize);
+ ret = btrfs_remove_free_space(cache, 3 * fsl + sectorsize,
+ fsl - sectorsize);
if (ret) {
test_err("error cleaning up tail 2 %d", ret);
goto out;
projects / linux / commitdiff