return ret;
}
+/*
+ * Align an offset up to the next full stripe boundary. Full stripe geometry
+ * is relative to the start of the block group, which is not necessarily
+ * full_stripe_len aligned in the logical address space, and full_stripe_len
+ * is not necessarily a power of two (e.g. 3 data stripes).
+ */
+static u64 stripe_run_align(const struct btrfs_block_group *block_group,
+ u64 offset)
+{
+ const u64 fsl = block_group->full_stripe_len;
+ u64 rel = offset - block_group->start + fsl - 1;
+
+ return div64_u64(rel, fsl) * fsl + block_group->start;
+}
+
+/*
+ * Search one bitmap entry for an aligned run of fully-free full stripes.
+ * Runs never extend beyond the bitmap entry, even if the neighbouring space
+ * is also free.
+ */
+static bool find_stripe_run_in_bitmap(struct btrfs_free_space_ctl *ctl,
+ struct btrfs_free_space *entry,
+ u64 want_bytes, u64 *rstart, u64 *rlen)
+{
+ struct btrfs_block_group *block_group = ctl->block_group;
+ const u32 unit = block_group->fs_info->sectorsize;
+ const u64 fsl = block_group->full_stripe_len;
+ const u64 bitmap_end = entry->offset + BITS_PER_BITMAP * unit;
+ u64 cand = stripe_run_align(block_group, entry->offset);
+
+ while (cand + fsl <= bitmap_end) {
+ unsigned long i = offset_to_bit(entry->offset, unit, cand);
+ unsigned long next_zero;
+ u64 run;
+
+ next_zero = find_next_zero_bit(entry->bitmap, BITS_PER_BITMAP,
+ i);
+ run = div64_u64((u64)(next_zero - i) * unit, fsl) * fsl;
+ if (run) {
+ *rstart = cand;
+ *rlen = min(want_bytes, run);
+ return true;
+ }
+ if (next_zero >= BITS_PER_BITMAP)
+ break;
+ cand = stripe_run_align(block_group,
+ entry->offset + (u64)(next_zero + 1) * unit);
+ }
+ return false;
+}
+
+/*
+ * Claim a contiguous run of fully-free full stripes.
+ *
+ * On success the run is removed from the free space cache and the caller owns
+ * [*start, *start + *len) exclusively until it either allocates from it or
+ * returns the unused remainder with btrfs_add_free_space().
+ *
+ * want_bytes must be a full_stripe_len multiple and is a maximum; the
+ * smallest successful claim is a single full stripe. A fully-free stripe
+ * split across two free space entries (e.g. an extent entry adjoining a
+ * bitmap) is not found: adjacent extent entries merge on insert, but
+ * extent/bitmap neighbours do not. This errs toward missing a claimable
+ * stripe, never toward claiming a non-free byte.
+ *
+ * Returns 0 on success, -ENOSPC if no aligned fully-free stripe is present
+ * in this block group.
+ */
+int btrfs_claim_free_stripe_run(struct btrfs_block_group *block_group,
+ u64 want_bytes, u64 *start, u64 *len)
+{
+ struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
+ struct btrfs_discard_ctl *discard_ctl =
+ &block_group->fs_info->discard_ctl;
+ const u64 fsl = block_group->full_stripe_len;
+ struct btrfs_free_space *entry;
+ struct rb_node *node;
+ enum btrfs_trim_state head_trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
+ u64 head_start = 0;
+ u64 head_len = 0;
+ u64 run_start;
+ u64 run_len;
+ int ret = -ENOSPC;
+
+ ASSERT(!btrfs_is_zoned(block_group->fs_info));
+ ASSERT(want_bytes >= fsl);
+ /* Claims are always whole stripes; tolerate an unaligned want_bytes. */
+ want_bytes = div64_u64(want_bytes, fsl) * fsl;
+
+ spin_lock(&ctl->tree_lock);
+
+ if (ctl->free_space < fsl)
+ goto out;
+
+ /*
+ * Walk the by-size index (largest max contiguous free run first)
+ * rather than the by-offset index. An entry whose largest contiguous
+ * free run is smaller than a full stripe cannot contain a fully-free
+ * stripe, and every following entry is no larger, so we stop at the
+ * first such entry. Near-full, where the free space degenerates into
+ * many sub-stripe holes, this is an O(1) fast fail instead of a scan
+ * of the whole free space tree on every allocation.
+ */
+ for (node = rb_first_cached(&ctl->free_space_bytes); node;
+ node = rb_next(node)) {
+ entry = rb_entry(node, struct btrfs_free_space, bytes_index);
+
+ if (get_max_extent_size(entry) < fsl)
+ break;
+
+ if (entry->bitmap) {
+ if (find_stripe_run_in_bitmap(ctl, entry, want_bytes,
+ &run_start, &run_len))
+ goto found;
+ continue;
+ }
+
+ run_start = stripe_run_align(block_group, entry->offset);
+ if (run_start + fsl > entry->offset + entry->bytes)
+ continue;
+ run_len = min(want_bytes,
+ div64_u64(entry->offset + entry->bytes - run_start,
+ fsl) * fsl);
+ goto found;
+ }
+ goto out;
+
+found:
+ if (!btrfs_free_space_trimmed(entry))
+ atomic64_add(run_len, &discard_ctl->discard_bytes_saved);
+
+ if (entry->bitmap) {
+ bitmap_clear_bits(ctl, entry, run_start, run_len, true);
+ if (!entry->bytes)
+ free_bitmap(ctl, entry);
+ } else {
+ const u64 end = entry->offset + entry->bytes;
+ int ret2;
+
+ unlink_free_space(ctl, entry, true);
+ head_start = entry->offset;
+ head_len = run_start - entry->offset;
+ head_trim_state = entry->trim_state;
+ entry->offset = run_start + run_len;
+ entry->bytes = end - entry->offset;
+ if (entry->bytes) {
+ ret2 = link_free_space(ctl, entry);
+ ASSERT(!ret2); /* -EEXIST; Logic error */
+ } else {
+ kmem_cache_free(btrfs_free_space_cachep, entry);
+ }
+ }
+ *start = run_start;
+ *len = run_len;
+ ret = 0;
+out:
+ btrfs_discard_update_discardable(block_group);
+ spin_unlock(&ctl->tree_lock);
+
+ if (head_len)
+ __btrfs_add_free_space(block_group, head_start, head_len,
+ head_trim_state);
+ return ret;
+}
+
/*
* given a cluster, put all of its extents back into the free space
* cache. If a block group is passed, this function will only free
return 0;
}
+/*
+ * Test btrfs_claim_free_stripe_run(): only aligned, fully-free, contiguous
+ * full-stripe runs may be claimed, geometry is relative to the block group
+ * start, and unaligned head/tail remainders stay in the free space cache.
+ * Stripe geometry is expressed in units of sectorsize so the test scales
+ * with the page-size-dependent test configs.
+ */
+static int test_stripe_claim(struct btrfs_block_group *cache, u32 sectorsize)
+{
+ const u64 orig_start = cache->start;
+ const u64 orig_fsl = cache->full_stripe_len;
+ /* 16 sectors per stripe, and a non-power-of-two variant. */
+ const u64 fsl = 16 * sectorsize;
+ const u64 fsl_np2 = 3 * sectorsize;
+ u64 start, len;
+ int ret;
+
+ test_msg("running stripe claim tests");
+ cache->full_stripe_len = fsl;
+
+ /* Empty cache: nothing to claim. */
+ ret = btrfs_claim_free_stripe_run(cache, fsl, &start, &len);
+ if (ret != -ENOSPC) {
+ test_err("claim from empty cache returned %d", ret);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Simple aligned claims, capped by want_bytes, then exhaustion. */
+ ret = btrfs_add_free_space(cache, 0, 4 * fsl);
+ if (ret) {
+ test_err("error adding free space %d", ret);
+ goto out;
+ }
+ ret = btrfs_claim_free_stripe_run(cache, 2 * fsl, &start, &len);
+ if (ret || start != 0 || len != 2 * fsl) {
+ test_err("first claim wrong: ret %d start %llu len %llu",
+ ret, start, len);
+ ret = -EINVAL;
+ goto out;
+ }
+ /* Remaining run is smaller than want_bytes: short claim. */
+ ret = btrfs_claim_free_stripe_run(cache, 4 * fsl, &start, &len);
+ if (ret || start != 2 * fsl || len != 2 * fsl) {
+ test_err("short claim wrong: ret %d start %llu len %llu",
+ ret, start, len);
+ ret = -EINVAL;
+ goto out;
+ }
+ ret = btrfs_claim_free_stripe_run(cache, fsl, &start, &len);
+ if (ret != -ENOSPC) {
+ test_err("claim from exhausted cache returned %d", ret);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* A partially-filled stripe is never claimed. */
+ ret = btrfs_add_free_space(cache, fsl + sectorsize,
+ 2 * fsl - sectorsize);
+ if (ret) {
+ test_err("error adding free space %d", ret);
+ goto out;
+ }
+ ret = btrfs_claim_free_stripe_run(cache, fsl, &start, &len);
+ if (ret || start != 2 * fsl || len != fsl) {
+ test_err("partial-stripe claim wrong: ret %d start %llu len %llu",
+ ret, start, len);
+ ret = -EINVAL;
+ goto out;
+ }
+ ret = btrfs_claim_free_stripe_run(cache, fsl, &start, &len);
+ if (ret != -ENOSPC) {
+ test_err("claimed a partially-filled stripe: %d", ret);
+ ret = -EINVAL;
+ goto out;
+ }
+ if (!test_check_exists(cache, fsl + sectorsize, fsl - sectorsize)) {
+ test_err("partial stripe tail went missing");
+ ret = -EINVAL;
+ goto out;
+ }
+ ret = btrfs_remove_free_space(cache, fsl + sectorsize,
+ fsl - sectorsize);
+ if (ret) {
+ test_err("error cleaning up %d", ret);
+ goto out;
+ }
+
+ /* Unaligned head remainder is preserved. */
+ ret = btrfs_add_free_space(cache, sectorsize, 3 * fsl - sectorsize);
+ if (ret) {
+ test_err("error adding free space %d", ret);
+ goto out;
+ }
+ ret = btrfs_claim_free_stripe_run(cache, 8 * fsl, &start, &len);
+ if (ret || start != fsl || len != 2 * fsl) {
+ test_err("head-carve claim wrong: ret %d start %llu len %llu",
+ ret, start, len);
+ ret = -EINVAL;
+ goto out;
+ }
+ if (!test_check_exists(cache, sectorsize, fsl - sectorsize)) {
+ test_err("head remainder went missing");
+ ret = -EINVAL;
+ goto out;
+ }
+ ret = btrfs_remove_free_space(cache, sectorsize, fsl - sectorsize);
+ if (ret) {
+ test_err("error cleaning up %d", ret);
+ goto out;
+ }
+
+ /* Stripe geometry is relative to the block group start. */
+ cache->start = 3 * sectorsize;
+ ret = btrfs_add_free_space(cache, cache->start + sectorsize,
+ 2 * fsl + sectorsize);
+ if (ret) {
+ test_err("error adding free space %d", ret);
+ goto out;
+ }
+ ret = btrfs_claim_free_stripe_run(cache, 4 * fsl, &start, &len);
+ if (ret || start != cache->start + fsl || len != fsl) {
+ test_err("bg-relative claim wrong: ret %d start %llu len %llu",
+ ret, start, len);
+ ret = -EINVAL;
+ goto out;
+ }
+ ret = btrfs_remove_free_space(cache, cache->start + sectorsize,
+ fsl - sectorsize);
+ if (ret) {
+ test_err("error cleaning up head %d", ret);
+ goto out;
+ }
+ ret = btrfs_remove_free_space(cache, cache->start + 2 * fsl,
+ 2 * sectorsize);
+ if (ret) {
+ test_err("error cleaning up tail %d", ret);
+ goto out;
+ }
+ cache->start = orig_start;
+
+ /* Same partial-stripe exclusion inside a bitmap entry. */
+ ret = test_add_free_space_entry(cache, fsl + sectorsize,
+ 2 * fsl - sectorsize, 1);
+ if (ret) {
+ test_err("error adding bitmap entry %d", ret);
+ goto out;
+ }
+ ret = btrfs_claim_free_stripe_run(cache, fsl, &start, &len);
+ if (ret || start != 2 * fsl || len != fsl) {
+ test_err("bitmap claim wrong: ret %d start %llu len %llu",
+ ret, start, len);
+ ret = -EINVAL;
+ goto out;
+ }
+ ret = btrfs_claim_free_stripe_run(cache, fsl, &start, &len);
+ if (ret != -ENOSPC) {
+ test_err("claimed a partially-filled bitmap stripe: %d", ret);
+ ret = -EINVAL;
+ goto out;
+ }
+ ret = btrfs_remove_free_space(cache, fsl + sectorsize,
+ fsl - sectorsize);
+ if (ret) {
+ test_err("error cleaning up bitmap %d", ret);
+ goto out;
+ }
+
+ /* Non-power-of-two full stripe length (e.g. 3 data stripes). */
+ cache->full_stripe_len = fsl_np2;
+ ret = btrfs_add_free_space(cache, sectorsize, 4 * fsl_np2);
+ if (ret) {
+ test_err("error adding free space %d", ret);
+ goto out;
+ }
+ ret = btrfs_claim_free_stripe_run(cache, 2 * fsl_np2, &start, &len);
+ if (ret || start != fsl_np2 || len != 2 * fsl_np2) {
+ test_err("np2 claim wrong: ret %d start %llu len %llu",
+ ret, start, len);
+ ret = -EINVAL;
+ goto out;
+ }
+ ret = btrfs_claim_free_stripe_run(cache, 2 * fsl_np2, &start, &len);
+ if (ret || start != 3 * fsl_np2 || len != fsl_np2) {
+ test_err("np2 tail claim wrong: ret %d start %llu len %llu",
+ ret, start, len);
+ ret = -EINVAL;
+ goto out;
+ }
+ ret = btrfs_remove_free_space(cache, sectorsize, fsl_np2 - sectorsize);
+ if (ret) {
+ test_err("error cleaning up np2 head %d", ret);
+ goto out;
+ }
+ ret = btrfs_remove_free_space(cache, 4 * fsl_np2, sectorsize);
+ if (ret) {
+ test_err("error cleaning up np2 tail %d", ret);
+ goto out;
+ }
+ ret = 0;
+out:
+ cache->start = orig_start;
+ cache->full_stripe_len = orig_fsl;
+ return ret;
+}
+
int btrfs_test_free_space_cache(u32 sectorsize, u32 nodesize)
{
struct btrfs_fs_info *fs_info;
if (ret)
goto out;
ret = test_bytes_index(cache, sectorsize);
+ if (ret)
+ goto out;
+ ret = test_stripe_claim(cache, sectorsize);
out:
btrfs_free_dummy_block_group(cache);
btrfs_free_dummy_root(root);
projects / linux / commitdiff