} else {
sinfo->bytes_stripe_open += delta;
}
+ /*
+ * A claimed run's remainder left the free space cache, but
+ * reloc_reserve_free() still counts it: move it out of a held
+ * group's share with it, or bytes_stripe_open and
+ * bytes_reloc_reserve both count it until the next commit's
+ * resync. Every writer of the share holds sinfo->lock.
+ */
+ btrfs_reloc_reserve_adjust(bg, -delta);
spin_unlock(&sinfo->lock);
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
* the counter can be made authoritative. Called after btrfs_finish_extent_commit
* so the free space cache reflects this transaction's frees.
*/
+static void reloc_reserve_resync(struct btrfs_block_group *bg);
+
void btrfs_scan_stripe_unusable(struct btrfs_fs_info *fs_info, bool force)
{
struct btrfs_space_info *sinfo;
total += bg->stripe_unusable;
total_stranded += READ_ONCE(bg->stripe_stranded);
total_claimable += bg->stripe_claimable;
- if (bg->start == READ_ONCE(fs_info->data_reloc_bg))
+ /* a held group's are in bytes_reloc_reserve */
+ if (bg->start == READ_ONCE(fs_info->data_reloc_bg) &&
+ !test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ &bg->runtime_flags))
total_claimable_reloc +=
bg->stripe_claimable;
nr_rw++;
largest = max(largest, bg->length);
+ if (test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ &bg->runtime_flags)) {
+ spin_lock(&bg->lock);
+ reloc_reserve_resync(bg);
+ spin_unlock(&bg->lock);
+ }
}
unit = max(unit, bg->full_stripe_len);
}
goto out;
}
+ /*
+ * A read-only group cannot be a relocation destination. It rejoins
+ * the reserve when it is writable again (btrfs_dec_block_group_ro()),
+ * or right away if it does not go read-only after all: a scrub or
+ * device replace makes a group read-only for a while, and a reserve
+ * group lost to every one of them was ordinary free space afterwards.
+ */
+ if (test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE, &cache->runtime_flags)) {
+ btrfs_reloc_reserve_clear(cache);
+ set_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE_LAPSED, &cache->runtime_flags);
+ }
+
num_bytes = btrfs_block_group_available_space(cache);
/*
if (btrfs_can_overcommit(sinfo, num_bytes - trapped,
BTRFS_RESERVE_NO_FLUSH, true))
ret = 0;
+ /*
+ * Or the relocation reserve can take this group's live tree
+ * blocks: a failed tree block allocation outside the reserve
+ * falls back into it. Only with a reserve and something to
+ * move: an empty group would pass "0 >= 0" with the reserve
+ * off. Data gets no such exemption: its test above fails only
+ * while bytes_may_use depends on this group's free space, and
+ * data writeback has no reserve to fall back on.
+ */
+ else if (sinfo->bytes_reloc_reserve &&
+ cache->used + cache->reserved &&
+ sinfo->bytes_reloc_reserve >=
+ cache->used + cache->reserved)
+ ret = 0;
}
if (!ret) {
list_add_tail(&cache->ro_list, &sinfo->ro_bgs);
}
out:
+ if (ret)
+ btrfs_reloc_reserve_rejoin(cache);
spin_unlock(&cache->lock);
spin_unlock(&sinfo->lock);
/*
((block_group->flags & BTRFS_BLOCK_GROUP_SYSTEM) &&
space_info->total_bytes - block_group->length <
system_space_target(space_info)) ||
- test_bit(BLOCK_GROUP_FLAG_FULLY_REMAPPED, &block_group->runtime_flags)) {
+ test_bit(BLOCK_GROUP_FLAG_FULLY_REMAPPED, &block_group->runtime_flags) ||
+ test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ &block_group->runtime_flags)) {
/*
* We want to bail if we made new allocations or have
* outstanding allocations in this block group. We do
sinfo->bytes_readonly -= btrfs_block_group_available_space(cache);
list_del_init(&cache->ro_list);
became_rw = true;
+ btrfs_reloc_reserve_rejoin(cache);
}
spin_unlock(&cache->lock);
spin_unlock(&sinfo->lock);
}
cache->reserved += num_bytes;
+ btrfs_reloc_reserve_adjust(cache, -(s64)num_bytes);
if (delalloc)
cache->delalloc_bytes += num_bytes;
spin_lock(&cache->lock);
bg_ro = cache->ro;
cache->reserved -= num_bytes;
+ if (!bg_ro && !btrfs_is_zoned(cache->fs_info))
+ btrfs_reloc_reserve_adjust(cache, num_bytes);
btrfs_maybe_reset_size_class(cache);
if (is_delalloc)
cache->delalloc_bytes -= num_bytes;
return 0;
}
+
+/*
+ * Relocation reserve.
+ *
+ * Relocating a block group needs somewhere to write its live data, and a
+ * filesystem short of free space is exactly when groups need relocating
+ * (reclaim, balance, device shrink or removal). So each space_info keeps a
+ * set of block groups held free for relocation alone: nothing else allocates
+ * from them (find_free_extent()), their free bytes count as used for every
+ * reservation but relocation's (bytes_reloc_reserve), statfs does not offer
+ * them, and the cleaner does not delete them while they are empty.
+ *
+ * Relocation moves at most one block group's live bytes, so the set holds at
+ * least the largest block group this space_info can have. Block groups vary in
+ * size (on raid56 the stripe width follows the devices that have free space),
+ * so the reserve is a set whose free bytes add up to that target rather than a
+ * single group. Relocation deletes its source group when it is done; the raw
+ * space that frees is claimed back for the reserve straight away, so the set
+ * refills from what relocation itself released. Groups leave the set, the
+ * least free first, only while the rest still covers the target.
+ */
+
+/* Free bytes of @bg; caller holds its lock. */
+static u64 reloc_reserve_free(const struct btrfs_block_group *bg)
+{
+ const u64 taken = bg->used + bg->reserved + bg->pinned +
+ bg->bytes_super + bg->zone_unusable;
+
+ return bg->length - min(bg->length, taken);
+}
+
+/* Caller holds space_info->lock and @bg->lock. */
+static void reloc_reserve_set(struct btrfs_block_group *bg)
+{
+ if (test_and_set_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE, &bg->runtime_flags))
+ return;
+ bg->reloc_reserve_bytes = reloc_reserve_free(bg);
+ /*
+ * An armed group's trapped free space is in bytes_stripe_unusable and
+ * its open runs' remainders in bytes_stripe_open: only its claimable
+ * stripes are relocation room (see reloc_reserve_resync()). A group
+ * not armed yet is capped when it is (btrfs_reloc_reserve_cap()).
+ */
+ if (READ_ONCE(bg->stripe_unusable_ready))
+ bg->reloc_reserve_bytes = min(bg->reloc_reserve_bytes,
+ READ_ONCE(bg->stripe_claimable));
+ bg->space_info->bytes_reloc_reserve += bg->reloc_reserve_bytes;
+}
+
+/* Claim a block group being created for the reserve; both locks held. */
+void btrfs_reloc_reserve_claim(struct btrfs_block_group *bg)
+{
+ reloc_reserve_set(bg);
+}
+
+/* Caller holds space_info->lock and @bg->lock. */
+void btrfs_reloc_reserve_clear(struct btrfs_block_group *bg)
+{
+ struct btrfs_space_info *sinfo = bg->space_info;
+
+ if (!test_and_clear_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ &bg->runtime_flags))
+ return;
+ sinfo->bytes_reloc_reserve -= min(sinfo->bytes_reloc_reserve,
+ bg->reloc_reserve_bytes);
+ bg->reloc_reserve_bytes = 0;
+}
+
+/*
+ * @delta bytes left (negative) or came back to (positive) the free space of
+ * @bg. Caller holds space_info->lock and @bg->lock.
+ */
+void btrfs_reloc_reserve_adjust(struct btrfs_block_group *bg, s64 delta)
+{
+ struct btrfs_space_info *sinfo = bg->space_info;
+ u64 d;
+
+ if (!test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE, &bg->runtime_flags))
+ return;
+ if (delta < 0) {
+ d = min_t(u64, bg->reloc_reserve_bytes, -delta);
+ bg->reloc_reserve_bytes -= d;
+ sinfo->bytes_reloc_reserve -= min(sinfo->bytes_reloc_reserve, d);
+ } else {
+ bg->reloc_reserve_bytes += delta;
+ sinfo->bytes_reloc_reserve += delta;
+ }
+}
+
+/*
+ * Lower @bg's share to at most @cap, for a group being armed for stripe_alloc
+ * or stripe_meta accounting: from then on only its claimable stripes are
+ * relocation room. Caller holds space_info->lock, which every writer of the
+ * share takes, but not @bg->lock.
+ */
+void btrfs_reloc_reserve_cap(struct btrfs_block_group *bg, u64 cap)
+{
+ lockdep_assert_held(&bg->space_info->lock);
+ if (test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE, &bg->runtime_flags) &&
+ bg->reloc_reserve_bytes > cap)
+ btrfs_reloc_reserve_adjust(bg,
+ -(s64)(bg->reloc_reserve_bytes - cap));
+}
+
+/*
+ * What the reserve must hold: the most live bytes any block group of
+ * @space_info can carry, its length less the superblock mirrors inside it. A
+ * relocation moves one existing group's live bytes, so the largest existing
+ * group bounds every relocation there is.
+ * (The chunk allocator's own caps are not a safe target: a new chunk can come
+ * out smaller than them when the devices are nearly full, and a target no
+ * chunk can reach would allocate a fresh one for the reserve on every retry.)
+ * The reserve's own groups count too, so a larger group appearing -- a new
+ * chunk over more devices, or under a larger chunk_size -- raises the target
+ * by itself. With no group at all, the allocator's cap stands in.
+ */
+u64 btrfs_reloc_reserve_target(struct btrfs_space_info *space_info)
+{
+ struct btrfs_fs_info *fs_info = space_info->fs_info;
+ struct btrfs_block_group *bg;
+ u64 target = 0;
+
+ down_read(&space_info->groups_sem);
+ for (int i = 0; i < BTRFS_NR_RAID_TYPES; i++)
+ list_for_each_entry(bg, &space_info->block_groups[i], list)
+ target = max(target, bg->length -
+ min(bg->length, bg->bytes_super));
+ up_read(&space_info->groups_sem);
+ if (!target)
+ target = min(READ_ONCE(space_info->chunk_size),
+ mult_perc(fs_info->fs_devices->total_rw_bytes, 10));
+ return target;
+}
+
+static bool reloc_reserve_wanted(struct btrfs_fs_info *fs_info,
+ struct btrfs_space_info *sinfo)
+{
+ if (btrfs_is_zoned(fs_info) || sb_rdonly(fs_info->sb) ||
+ BTRFS_FS_ERROR(fs_info) || btrfs_fs_closing(fs_info))
+ return false;
+ /*
+ * System and remap tree chunks are tiny and relocated through their
+ * own path.
+ */
+ if (sinfo->flags & (BTRFS_BLOCK_GROUP_SYSTEM |
+ BTRFS_BLOCK_GROUP_METADATA_REMAP))
+ return false;
+ if (btrfs_mixed_space_info(sinfo))
+ return false;
+ return READ_ONCE(sinfo->reloc_reserve);
+}
+
+/*
+ * Under stripe_alloc / stripe_meta only a held group's whole claimable
+ * stripes are relocation room; its trapped free space is counted in
+ * bytes_stripe_unusable already, and counting it here too made the reserve
+ * look full while relocation found no stripe to claim (claimable 3.3M
+ * against a 34M reserve). Resync the group's share to its claimable
+ * stripes at each commit-time rescan. Caller holds space_info->lock and
+ * @bg->lock.
+ */
+static void reloc_reserve_resync(struct btrfs_block_group *bg)
+{
+ struct btrfs_space_info *sinfo = bg->space_info;
+ u64 want;
+
+ if (!test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE, &bg->runtime_flags) ||
+ !READ_ONCE(bg->stripe_unusable_ready))
+ return;
+ want = min(reloc_reserve_free(bg), READ_ONCE(bg->stripe_claimable));
+ sinfo->bytes_reloc_reserve -= min(sinfo->bytes_reloc_reserve,
+ bg->reloc_reserve_bytes);
+ bg->reloc_reserve_bytes = want;
+ sinfo->bytes_reloc_reserve += want;
+}
+
+/*
+ * @bg left the reserve to go read-only and is writable again (or never went
+ * read-only): rejoin, unless the reserve is off. A surplus this makes is
+ * released by the next settle. Caller holds space_info->lock and @bg->lock.
+ */
+void btrfs_reloc_reserve_rejoin(struct btrfs_block_group *bg)
+{
+ if (!test_and_clear_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE_LAPSED,
+ &bg->runtime_flags))
+ return;
+ if (!reloc_reserve_wanted(bg->fs_info, bg->space_info))
+ return;
+ reloc_reserve_set(bg);
+}
+
+/*
+ * Bring @sinfo's reserve to its target: take empty groups (largest first)
+ * until it is covered, then release groups (least free first) while the rest
+ * still covers it -- in that order, so a small group taken while short is
+ * given back as soon as a larger one arrives. With @wanted false everything
+ * is released. Returns the shortfall.
+ */
+static u64 reloc_reserve_settle(struct btrfs_space_info *sinfo, u64 target,
+ bool wanted)
+{
+ struct btrfs_block_group *bg;
+ struct btrfs_block_group *pick;
+ /*
+ * Released groups that are empty go back to the cleaner, which skipped
+ * them while they were held; only after the locks are dropped.
+ */
+ struct btrfs_block_group *released[8];
+ int nr_released = 0;
+ u64 short_by;
+
+ down_read(&sinfo->groups_sem);
+ spin_lock(&sinfo->lock);
+ while (wanted && sinfo->bytes_reloc_reserve < target) {
+ pick = NULL;
+ for (int i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
+ list_for_each_entry(bg, &sinfo->block_groups[i], list) {
+ if (test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ &bg->runtime_flags) ||
+ test_bit(BLOCK_GROUP_FLAG_REMOVED,
+ &bg->runtime_flags) ||
+ test_bit(BLOCK_GROUP_FLAG_FULLY_REMAPPED,
+ &bg->runtime_flags) ||
+ (bg->flags & BTRFS_BLOCK_GROUP_REMAPPED))
+ continue;
+ spin_lock(&bg->lock);
+ if (!bg->ro && !btrfs_is_block_group_used(bg) &&
+ (!pick || bg->length > pick->length))
+ pick = bg;
+ spin_unlock(&bg->lock);
+ }
+ }
+ if (!pick)
+ break;
+ spin_lock(&pick->lock);
+ if (!pick->ro && !btrfs_is_block_group_used(pick))
+ reloc_reserve_set(pick);
+ spin_unlock(&pick->lock);
+ }
+ while (nr_released < ARRAY_SIZE(released)) {
+ pick = NULL;
+ for (int i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
+ list_for_each_entry(bg, &sinfo->block_groups[i], list) {
+ if (!test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ &bg->runtime_flags))
+ continue;
+ if (wanted && sinfo->bytes_reloc_reserve -
+ min(sinfo->bytes_reloc_reserve,
+ bg->reloc_reserve_bytes) < target)
+ continue;
+ if (!pick || bg->reloc_reserve_bytes <
+ pick->reloc_reserve_bytes)
+ pick = bg;
+ }
+ }
+ if (!pick)
+ break;
+ spin_lock(&pick->lock);
+ btrfs_reloc_reserve_clear(pick);
+ if (!btrfs_is_block_group_used(pick)) {
+ btrfs_get_block_group(pick);
+ released[nr_released++] = pick;
+ }
+ spin_unlock(&pick->lock);
+ }
+ short_by = target - min(target, sinfo->bytes_reloc_reserve);
+ if (!short_by)
+ sinfo->reloc_reserve_warned = false;
+ spin_unlock(&sinfo->lock);
+ up_read(&sinfo->groups_sem);
+ for (int i = 0; i < nr_released; i++) {
+ btrfs_mark_bg_unused(released[i]);
+ btrfs_put_block_group(released[i]);
+ }
+ return short_by;
+}
+
+static void reloc_reserve_update_one(struct btrfs_fs_info *fs_info,
+ struct btrfs_space_info *sinfo,
+ bool may_alloc)
+{
+ const bool wanted = reloc_reserve_wanted(fs_info, sinfo);
+ struct btrfs_trans_handle *trans;
+ u64 target = wanted ? btrfs_reloc_reserve_target(sinfo) : 0;
+ u64 short_by;
+ int ret;
+
+ short_by = reloc_reserve_settle(sinfo, target, wanted);
+ /*
+ * A new chunk can come out a little smaller than the largest group
+ * (devices nearly full, stripe rounding): allocating another whole
+ * chunk to cover a few megabytes would hold twice the target.
+ */
+ if (short_by <= min(target / 32, (u64)SZ_64M))
+ short_by = 0;
+ if (!short_by || !may_alloc)
+ return;
+ if (time_before(jiffies, READ_ONCE(sinfo->reloc_reserve_retry)))
+ return;
+
+ /*
+ * A device shrink or removal relocates to free space: growing the
+ * reserve back with new chunks meanwhile takes the room it is freeing
+ * (a raid6 shrink failed in merge_reloc_roots with -ENOSPC and forced
+ * the filesystem read-only). Take only empty groups until it is done;
+ * the cleaner refills after it.
+ */
+ switch (READ_ONCE(fs_info->exclusive_operation)) {
+ case BTRFS_EXCLOP_RESIZE:
+ case BTRFS_EXCLOP_DEV_REMOVE:
+ return;
+ default:
+ break;
+ }
+
+ /* No empty group to take: make one while unallocated space exists. */
+ trans = btrfs_join_transaction(fs_info->tree_root);
+ if (IS_ERR(trans))
+ return;
+ spin_lock(&sinfo->lock);
+ sinfo->reloc_reserve_claim_next = true;
+ spin_unlock(&sinfo->lock);
+ ret = btrfs_chunk_alloc(trans, sinfo,
+ btrfs_get_alloc_profile(fs_info, sinfo->flags),
+ CHUNK_ALLOC_FORCE);
+ spin_lock(&sinfo->lock);
+ sinfo->reloc_reserve_claim_next = false;
+ spin_unlock(&sinfo->lock);
+ btrfs_end_transaction(trans);
+ if (ret > 0)
+ short_by = reloc_reserve_settle(sinfo, target, wanted);
+ if (short_by <= min(target / 32, (u64)SZ_64M))
+ return;
+ WRITE_ONCE(sinfo->reloc_reserve_retry, jiffies + 60 * HZ);
+ if (!sinfo->reloc_reserve_warned) {
+ sinfo->reloc_reserve_warned = true;
+ btrfs_info(fs_info,
+"relocation reserve for %s block groups is %llu bytes short of %llu; relocation may fail with ENOSPC until space is freed",
+ (sinfo->flags & BTRFS_BLOCK_GROUP_DATA) ? "data" : "metadata",
+ short_by, target);
+ }
+}
+
+/*
+ * Establish, refill or release the relocation reserve of every space_info.
+ * @may_alloc: a chunk may be allocated (not while holding a transaction).
+ */
+void btrfs_reloc_reserve_update(struct btrfs_fs_info *fs_info, bool may_alloc)
+{
+ struct btrfs_space_info *sinfo;
+
+ list_for_each_entry(sinfo, &fs_info->space_info, list)
+ reloc_reserve_update_one(fs_info, sinfo, may_alloc);
+}
+
+/*
+ * Refill the reserve after relocation removed its source group. The group's
+ * device extents can be allocated again only once the removal commits: a
+ * refill before that found only the leftovers and claimed a sliver of a chunk
+ * (37M, where the relocated group had released 864M), and the next relocation
+ * in the same balance failed with ENOSPC. Commit first, when a reserve is on.
+ */
+void btrfs_reloc_reserve_refill_after_removal(struct btrfs_fs_info *fs_info)
+{
+ struct btrfs_space_info *sinfo;
+ struct btrfs_trans_handle *trans;
+ bool wanted = false;
+
+ list_for_each_entry(sinfo, &fs_info->space_info, list)
+ wanted |= reloc_reserve_wanted(fs_info, sinfo);
+ if (!wanted)
+ return;
+ trans = btrfs_join_transaction(fs_info->tree_root);
+ if (IS_ERR(trans))
+ return;
+ if (btrfs_commit_transaction(trans))
+ return;
+ btrfs_reloc_reserve_update(fs_info, true);
+}
* write-hole debug check skips such groups (sticky, debug only).
*/
BLOCK_GROUP_FLAG_STRIPE_NOCOW_USED,
+ /*
+ * Held empty for relocation: only relocation may allocate here (see
+ * btrfs_reloc_reserve_update()).
+ */
+ BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ /*
+ * Left the relocation reserve to go read-only (scrub, replace, a
+ * relocation that failed): rejoins it when writable again.
+ */
+ BLOCK_GROUP_FLAG_RELOC_RESERVE_LAPSED,
/*
* A RAID5/6 group with a single data stripe, which is mapped as
* RAID1/RAID1C3 (its chunk map's type differs from on_disk_type). Its
*/
u64 alloc_offset;
u64 zone_unusable;
+ /*
+ * While BLOCK_GROUP_FLAG_RELOC_RESERVE is set: this group's share of
+ * space_info->bytes_reloc_reserve (its free bytes; under stripe_alloc
+ * or stripe_meta, its claimable stripes). Written under
+ * space_info->lock, mostly with this group's lock too; read under
+ * either.
+ */
+ u64 reloc_reserve_bytes;
u64 zone_capacity;
u64 meta_write_pointer;
struct btrfs_chunk_map *physical_map;
int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
struct btrfs_chunk_map *map);
void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info);
+void btrfs_reloc_reserve_update(struct btrfs_fs_info *fs_info, bool may_alloc);
+void btrfs_reloc_reserve_refill_after_removal(struct btrfs_fs_info *fs_info);
+void btrfs_reloc_reserve_clear(struct btrfs_block_group *bg);
+void btrfs_reloc_reserve_rejoin(struct btrfs_block_group *bg);
+void btrfs_reloc_reserve_claim(struct btrfs_block_group *bg);
+void btrfs_reloc_reserve_adjust(struct btrfs_block_group *bg, s64 delta);
+void btrfs_reloc_reserve_cap(struct btrfs_block_group *bg, u64 cap);
+u64 btrfs_reloc_reserve_target(struct btrfs_space_info *space_info);
void btrfs_mark_bg_unused(struct btrfs_block_group *bg);
void btrfs_reclaim_block_groups(struct btrfs_fs_info *fs_info, unsigned int limit);
void btrfs_reclaim_bgs_work(struct work_struct *work);
*/
btrfs_delete_unused_bgs(fs_info);
+ /* Keep the relocation reserve at its target. */
+ btrfs_reloc_reserve_update(fs_info, true);
+
/*
* Reclaim block groups in the reclaim_bgs list after we deleted
* all unused block_groups. This possibly gives us some more free
up_read(&fs_info->cleanup_work_sem);
WRITE_ONCE(fs_info->mount_cleanup_task, NULL);
+ /* Log replay and orphan cleanup are done: hold the relocation reserve. */
+ btrfs_reloc_reserve_update(fs_info, true);
+
mutex_lock(&fs_info->cleaner_mutex);
ret = btrfs_recover_relocation(fs_info);
mutex_unlock(&fs_info->cleaner_mutex);
/* Need reset before reusing in a zoned block group */
btrfs_space_info_update_bytes_zone_unusable(space_info, len);
} else if (return_free_space) {
+ spin_lock(&cache->lock);
+ btrfs_reloc_reserve_adjust(cache, len);
+ spin_unlock(&cache->lock);
btrfs_return_free_space(space_info, len);
}
if (quarantine_back)
!block_group_bits(cluster_bg, ffe_ctl->flags) ||
!find_free_extent_check_size_class(ffe_ctl, cluster_bg)))
goto release_cluster;
+ /* A cluster may still sit in a group since reserved for relocation. */
+ if (!ffe_ctl->for_reloc &&
+ test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE, &cluster_bg->runtime_flags))
+ goto release_cluster;
offset = btrfs_alloc_from_cluster(cluster_bg, last_ptr,
ffe_ctl->num_bytes, cluster_bg->start,
}
spin_unlock(&fs_info->relocation_bg_lock);
if (dedicated_now && block_group->space_info) {
- /* its claimable stripes now belong to relocation alone */
+ /*
+ * its claimable stripes now belong to relocation alone; a held
+ * group's are in bytes_reloc_reserve, not to be subtracted
+ * from everyone else's supply twice. A group that joins the
+ * reserve after its dedication drops out at the next rescan.
+ */
spin_lock(&block_group->space_info->lock);
block_group->space_info->bytes_stripe_claimable_reloc =
+ test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ &block_group->runtime_flags) ? 0 :
READ_ONCE(block_group->stripe_claimable);
spin_unlock(&block_group->space_info->lock);
}
down_read(&space_info->groups_sem);
if (list_empty(&block_group->list) ||
block_group->ro ||
- (block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED)) {
+ (block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED) ||
+ (!ffe_ctl->for_reloc &&
+ test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ &block_group->runtime_flags))) {
/*
* someone is removing this block group,
* we can't jump into the have_block_group
continue;
}
+ /* Held for relocation; nobody else may allocate here. */
+ if (!ffe_ctl->for_reloc &&
+ test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ &block_group->runtime_flags))
+ continue;
+
btrfs_grab_block_group(block_group, ffe_ctl->delalloc);
ffe_ctl->search_start = block_group->start;
u64 log_owner = for_treelog ? root->root_key.offset : 0;
u64 log_transid = for_treelog ? root->log_transid : 0;
bool for_data_reloc = (btrfs_is_data_reloc_root(root) && is_data);
+ bool use_reloc_reserve = false;
struct btrfs_inode *steer_inode = NULL;
enum btrfs_stripe_run_class steer_class = BTRFS_STRIPE_RUN_COW;
ffe_ctl.steer_inode = steer_inode;
ffe_ctl.steer_class = steer_class;
ffe_ctl.split_bytes = split_bytes;
+ ffe_ctl.for_reloc = for_data_reloc || btrfs_is_reloc_task(fs_info) ||
+ use_reloc_reserve;
ret = find_free_extent(root, ins, &ffe_ctl);
if (!ret && !is_data) {
btrfs_dec_block_group_reservations(fs_info, ins->objectid);
} else if (ret == -ENOSPC) {
+ /*
+ * A tree block was admitted by the reservation layer before it
+ * got here, and failing its allocation aborts the transaction.
+ * The relocation reserve limits admission, not this: relocating
+ * a metadata group leaves delayed refs and commit work that
+ * other tasks run, and their tree blocks may only find room in
+ * the reserve. Taking it costs relocation some headroom; an
+ * abort would cost the filesystem.
+ */
+ if (!is_data && !ffe_ctl.for_reloc) {
+ use_reloc_reserve = true;
+ goto again;
+ }
if (!final_tried && ins->offset) {
num_bytes = min(num_bytes >> 1, ins->offset);
num_bytes = round_down(num_bytes,
/* Allocation is called for data relocation */
bool for_data_reloc;
+ /* Relocation's allocation: may use the relocation reserve. */
+ bool for_reloc;
+
/*
* Data allocation to steer into a private stripe run (see
* stripe_run_owner()), and the run class: NOCOW for nodatacow and
sinfo->bytes_stripe_claimable = 0;
else
btrfs_space_info_update_bytes_stripe_claimable(sinfo, delta);
- if (bg->start == READ_ONCE(bg->fs_info->data_reloc_bg)) {
+ /* a held group's stripes are in bytes_reloc_reserve instead */
+ if (bg->start == READ_ONCE(bg->fs_info->data_reloc_bg) &&
+ !test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE, &bg->runtime_flags)) {
if (delta < 0 && sinfo->bytes_stripe_claimable_reloc < (u64)-delta)
sinfo->bytes_stripe_claimable_reloc = 0;
else
btrfs_space_info_update_bytes_stripe_unusable(bg->space_info, trapped);
btrfs_space_info_update_bytes_stripe_claimable(bg->space_info,
claimable);
+ /* A held group's share is its claimable stripes from here on. */
+ btrfs_reloc_reserve_cap(bg, claimable);
spin_unlock(&bg->space_info->lock);
spin_unlock(&ctl->tree_lock);
kvfree(freep);
#define btrfs_test_opt(fs_info, opt) ((fs_info)->mount_opt & \
BTRFS_MOUNT_##opt)
+/* Is the current task the one running a block group relocation? */
+static inline bool btrfs_is_reloc_task(const struct btrfs_fs_info *fs_info)
+{
+ return READ_ONCE(fs_info->reloc_owner) == current;
+}
+
static inline bool btrfs_is_full_ro(const struct btrfs_fs_info *fs_info)
{
if (!sb_rdonly(fs_info->sb))
bool steal;
/* Made by the relocation task: may draw on bytes_stripe_reserve. */
bool reloc;
+ /* Made by the relocating task: may draw on the relocation reserve. */
+ bool reloc_reserve;
/* Made by the mount-time orphan cleanup: may steal regardless. */
bool mount_cleanup;
/*
spinlock_t lock;
};
+/*
+ * The relocation reserve counts as used for everyone but relocation: for the
+ * relocating task it is the room it moves live extents into.
+ */
+static u64 reloc_adjusted_used(const struct btrfs_space_info *s_info, u64 used,
+ bool reloc)
+{
+ if (reloc)
+ used -= min(used, s_info->bytes_reloc_reserve);
+ return used;
+}
+
/*
* after adding space to the filesystem, we need to clear the full flags
* on all the space infos.
INIT_LIST_HEAD(&space_info->ro_bgs);
INIT_LIST_HEAD(&space_info->tickets);
INIT_LIST_HEAD(&space_info->priority_tickets);
+ /*
+ * Off until enabled through sysfs (allocation/<type>/reloc_reserve):
+ * it costs capacity, and changes when relocation can run at all.
+ */
+ space_info->reloc_reserve = false;
space_info->clamp = 1;
btrfs_update_space_info_chunk_size(space_info, calc_chunk_size(info, flags));
space_info->subgroup_id = BTRFS_SUB_GROUP_PRIMARY;
btrfs_space_info_update_bytes_zone_unusable(space_info, block_group->zone_unusable);
if (block_group->length > 0)
space_info->full = false;
+ if (space_info->reloc_reserve_claim_next &&
+ test_bit(BLOCK_GROUP_FLAG_NEW, &block_group->runtime_flags)) {
+ space_info->reloc_reserve_claim_next = false;
+ spin_lock(&block_group->lock);
+ btrfs_reloc_reserve_claim(block_group);
+ spin_unlock(&block_group->lock);
+ }
btrfs_try_granting_tickets(space_info);
spin_unlock(&space_info->lock);
*/
if (reloc)
used -= min(used, space_info->bytes_stripe_reserve);
+ /* the relocation reserve is the relocating task's alone */
+ used = reloc_adjusted_used(space_info, used, reloc &&
+ btrfs_is_reloc_task(space_info->fs_info));
return check_can_overcommit(space_info, used, bytes, flush);
}
} else {
/* ...and that group is out of reach for everyone else */
supply -= min(supply, dedicated);
+ /*
+ * ...and so are the groups held for relocation (the
+ * relocation reserve): whole stripes, but relocation's.
+ */
+ supply -= min(supply,
+ READ_ONCE(space_info->bytes_reloc_reserve));
/*
* Charge what writeback can claim for the data: whole
* stripes of whichever group it lands in. Not the
tused = used;
if (ticket->reloc)
tused -= min(tused, space_info->bytes_stripe_reserve);
+ tused = reloc_adjusted_used(space_info, tused,
+ ticket->reloc_reserve);
/*
* Check and see if our ticket can be satisfied now. The
flush == BTRFS_RESERVE_NO_FLUSH ||
flush == BTRFS_RESERVE_FLUSH_LIMIT ||
flush == BTRFS_RESERVE_FLUSH_EMERGENCY;
+ /* Only the relocating task may draw on the relocation reserve. */
+ const bool reloc_task = btrfs_is_reloc_task(fs_info);
ASSERT(orig_bytes, "orig_bytes=%llu", orig_bytes);
/*
async_work = &fs_info->async_reclaim_work;
spin_lock(&space_info->lock);
- used = btrfs_space_info_used(space_info, true);
+ used = reloc_adjusted_used(space_info,
+ btrfs_space_info_used(space_info, true), reloc_task);
/*
* We don't want NO_FLUSH allocations to jump everybody, they can
spin_lock_init(&ticket.lock);
ticket.steal = can_steal(flush);
ticket.reloc = reloc;
+ ticket.reloc_reserve = reloc_task;
ticket.mount_cleanup =
READ_ONCE(fs_info->mount_cleanup_task) == current;
ticket.hold_stripe = held != NULL;
dump_global_block_rsv(fs_info);
}
+/*
+ * Raw bytes held free in groups reserved for relocation, for statfs: that
+ * space cannot be written by anyone but relocation.
+ */
+u64 btrfs_account_reloc_reserve_free_space(struct btrfs_space_info *sinfo)
+{
+ struct btrfs_block_group *bg;
+ u64 free_bytes = 0;
+
+ if (!sinfo->bytes_reloc_reserve)
+ return 0;
+ down_read(&sinfo->groups_sem);
+ for (int i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
+ list_for_each_entry(bg, &sinfo->block_groups[i], list) {
+ spin_lock(&bg->lock);
+ if (test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ &bg->runtime_flags))
+ free_bytes += bg->reloc_reserve_bytes *
+ btrfs_bg_type_to_factor(bg->flags);
+ spin_unlock(&bg->lock);
+ }
+ }
+ up_read(&sinfo->groups_sem);
+ return free_bytes;
+}
+
/*
* Account the unused space of all the readonly block group in the space_info.
* takes mirrors into account.
u64 bytes_may_use; /* number of bytes that may be used for
delalloc/allocations */
u64 bytes_readonly; /* total bytes that are read only */
+ u64 bytes_reloc_reserve; /* free bytes in groups held empty for
+ relocation; used for everyone else */
u64 bytes_zone_unusable; /* total bytes that are unusable until
resetting the device zone */
u64 bytes_stripe_unusable; /* free bytes stripe_alloc and
*/
int bg_reclaim_threshold;
+ /* Keep a relocation reserve (sysfs reloc_reserve). */
+ bool reloc_reserve;
+ /* Said once that the reserve could not be established. */
+ bool reloc_reserve_warned;
+ /* No chunk allocation for the reserve before this (jiffies). */
+ unsigned long reloc_reserve_retry;
+ /*
+ * The next block group created here goes to the reserve as it is
+ * added, before anything can allocate from it: a chunk made for the
+ * reserve would otherwise take the tree blocks of its own creation, or
+ * the writes of whoever was waiting, and never be empty.
+ */
+ bool reloc_reserve_claim_next;
+
int clamp; /* Used to scale our threshold for preemptive
flushing. The value is >> clamp, so turns
out to be a 2^clamp divisor. */
return s_info->bytes_used + s_info->bytes_reserved +
s_info->bytes_pinned + s_info->bytes_readonly +
- s_info->bytes_zone_unusable +
+ s_info->bytes_zone_unusable + s_info->bytes_reloc_reserve +
s_info->bytes_stripe_unusable + s_info->bytes_stripe_open +
s_info->bytes_stripe_margin +
s_info->bytes_stripe_reserve +
u64 orig_bytes,
enum btrfs_reserve_flush_enum flush);
void btrfs_try_granting_tickets(struct btrfs_space_info *space_info);
+u64 btrfs_account_reloc_reserve_free_space(struct btrfs_space_info *sinfo);
bool btrfs_can_overcommit(const struct btrfs_space_info *space_info, u64 bytes,
enum btrfs_reserve_flush_enum flush, bool reloc);
total_free_data += found->disk_total - found->disk_used;
total_free_data -=
btrfs_account_ro_block_groups_free_space(found);
+ total_free_data -=
+ btrfs_account_reloc_reserve_free_space(found);
for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
if (!list_empty(&found->block_groups[i]))
SPACE_INFO_ATTR(bytes_stripe_margin);
SPACE_INFO_ATTR(bytes_stripe_reserve);
SPACE_INFO_ATTR(bytes_stripe_quarantine);
+SPACE_INFO_ATTR(bytes_reloc_reserve);
SPACE_INFO_ATTR(disk_used);
SPACE_INFO_ATTR(disk_total);
SPACE_INFO_ATTR(reclaim_count);
btrfs_sinfo_bg_reclaim_threshold_show,
btrfs_sinfo_bg_reclaim_threshold_store);
+static ssize_t btrfs_sinfo_reloc_reserve_show(struct kobject *kobj,
+ struct kobj_attribute *a,
+ char *buf)
+{
+ struct btrfs_space_info *space_info = to_space_info(kobj);
+
+ return sysfs_emit(buf, "%d\n", READ_ONCE(space_info->reloc_reserve));
+}
+
+static ssize_t btrfs_sinfo_reloc_reserve_store(struct kobject *kobj,
+ struct kobj_attribute *a,
+ const char *buf, size_t len)
+{
+ struct btrfs_space_info *space_info = to_space_info(kobj);
+ bool val;
+ int ret;
+
+ ret = kstrtobool(buf, &val);
+ if (ret)
+ return ret;
+ if (val && btrfs_is_zoned(space_info->fs_info))
+ return -EINVAL;
+ WRITE_ONCE(space_info->reloc_reserve, val);
+ WRITE_ONCE(space_info->reloc_reserve_retry, jiffies);
+ btrfs_reloc_reserve_update(space_info->fs_info, false);
+ return len;
+}
+
+BTRFS_ATTR_RW(space_info, reloc_reserve, btrfs_sinfo_reloc_reserve_show,
+ btrfs_sinfo_reloc_reserve_store);
+
+static ssize_t btrfs_sinfo_reloc_reserve_target_show(struct kobject *kobj,
+ struct kobj_attribute *a,
+ char *buf)
+{
+ struct btrfs_space_info *space_info = to_space_info(kobj);
+
+ return sysfs_emit(buf, "%llu\n",
+ btrfs_reloc_reserve_target(space_info));
+}
+
+BTRFS_ATTR(space_info, reloc_reserve_target,
+ btrfs_sinfo_reloc_reserve_target_show);
+
static ssize_t btrfs_sinfo_dynamic_reclaim_show(struct kobject *kobj,
struct kobj_attribute *a,
char *buf)
BTRFS_ATTR_PTR(space_info, disk_used),
BTRFS_ATTR_PTR(space_info, disk_total),
BTRFS_ATTR_PTR(space_info, bg_reclaim_threshold),
+ BTRFS_ATTR_PTR(space_info, bytes_reloc_reserve),
+ BTRFS_ATTR_PTR(space_info, reloc_reserve),
+ BTRFS_ATTR_PTR(space_info, reloc_reserve_target),
BTRFS_ATTR_PTR(space_info, dynamic_reclaim),
BTRFS_ATTR_PTR(space_info, chunk_size),
BTRFS_ATTR_PTR(space_info, size_classes),
bg->start);
}
+ /*
+ * Removing the emptied group is the last step of the relocation and
+ * reserves metadata like the rest of it: on a full filesystem an
+ * ordinary reservation here fails after all the data has moved. The
+ * caller holds reclaim_bgs_lock, so this is still the only relocation.
+ */
+ WRITE_ONCE(fs_info->reloc_owner, current);
trans = btrfs_start_trans_remove_block_group(root->fs_info, bg->start);
if (IS_ERR(trans)) {
+ WRITE_ONCE(fs_info->reloc_owner, NULL);
ret = PTR_ERR(trans);
btrfs_handle_fs_error(root->fs_info, ret, NULL);
return ret;
/* Step two, delete the device extents and the chunk tree entries. */
ret = btrfs_remove_chunk(trans, bg->start);
btrfs_end_transaction(trans);
+ WRITE_ONCE(fs_info->reloc_owner, NULL);
+ /* Claim the space the source group released back for the reserve. */
+ if (!ret)
+ btrfs_reloc_reserve_refill_after_removal(fs_info);
return ret;
}
return false;
}
+ /*
+ * A group held for relocation is empty by design: relocating it moves
+ * nothing and only deletes the reserve. A profile conversion still
+ * has to take it.
+ */
+ if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT)) {
+ struct btrfs_block_group *bg;
+ bool reserved = false;
+
+ bg = btrfs_lookup_block_group(fs_info, chunk_offset);
+ if (bg) {
+ reserved = test_bit(BLOCK_GROUP_FLAG_RELOC_RESERVE,
+ &bg->runtime_flags);
+ btrfs_put_block_group(bg);
+ }
+ if (reserved)
+ return false;
+ }
+
/* usage filter */
if (bargs->flags & BTRFS_BALANCE_ARGS_USAGE) {
int ret2;