u64 old_total;
u64 trapped;
int ret = 0;
+ u8 forced = READ_ONCE(bg->stripe_reclaim_forced);
bool compact = false;
bool compacted = false;
u64 need = READ_ONCE(bg->used);
/* Compaction moves at most half of it. */
- if (btrfs_stripe_bg_wants_compaction(bg))
+ if (forced ? forced == STRIPE_RECLAIM_COMPACT :
+ btrfs_stripe_bg_wants_compaction(bg))
need /= 2;
elsewhere -= min(elsewhere, READ_ONCE(bg->stripe_claimable));
if (elsewhere < need) {
spin_lock(&space_info->lock);
spin_lock(&bg->lock);
+ /* A forced reclaim is done once, or dropped with the group. */
+ forced = bg->stripe_reclaim_forced;
+ bg->stripe_reclaim_forced = STRIPE_RECLAIM_NONE;
if (bg->ro ||
((bg->reserved || bg->pinned) &&
!btrfs_is_stripe_alloc_bg(bg))) {
* check, which is more meaningful when adding to the list in
* the first place.
*/
- if (!should_reclaim_block_group(bg, bg->length)) {
+ if (forced) {
+ /* stripe_reclaim_now chose already; the thresholds may say no. */
+ compact = (forced == STRIPE_RECLAIM_COMPACT);
+ } else if (!should_reclaim_block_group(bg, bg->length)) {
/*
* Too full to relocate whole. A stripe_alloc
* group whose free space is trapped in partially used stripes
return true;
}
+/*
+ * Room for a forced reclaim beyond the other groups' claimable stripes: the
+ * one chunk the reclaim worker allocates when they fall short (see
+ * btrfs_reclaim_block_group()), estimated as the data share of the
+ * unallocated device space at @bg's width, capped at @bg's length.
+ */
+static u64 stripe_reclaim_chunk_room(struct btrfs_block_group *bg)
+{
+ struct btrfs_fs_info *fs_info = bg->fs_info;
+ struct btrfs_chunk_map *map;
+ u64 unalloc = atomic64_read(&fs_info->free_chunk_space);
+ u64 room;
+
+ map = btrfs_get_chunk_map(fs_info, bg->start, 1);
+ if (IS_ERR(map))
+ return 0;
+ room = div_u64(unalloc * nr_data_stripes(map), map->num_stripes);
+ btrfs_free_chunk_map(map);
+ return min(room, bg->length);
+}
+
+/*
+ * Queue @bg for the reclaim worker with the reclaim it is to do. Returns
+ * false when the group is already on a list (queued, unused or new).
+ */
+static bool stripe_reclaim_now_queue(struct btrfs_block_group *bg,
+ enum stripe_reclaim_kind kind)
+{
+ bool queued;
+
+ spin_lock(&bg->lock);
+ bg->stripe_reclaim_forced = kind;
+ queued = btrfs_mark_bg_to_reclaim(bg);
+ if (queued)
+ bg->stripe_reclaim_jiffies = jiffies ?: 1;
+ else
+ bg->stripe_reclaim_forced = STRIPE_RECLAIM_NONE;
+ spin_unlock(&bg->lock);
+ if (queued)
+ btrfs_debug(bg->fs_info,
+ "reclaim: queued block group %llu to %s (stripe_reclaim_now)",
+ bg->start, kind == STRIPE_RECLAIM_COMPACT ?
+ "compact" : "relocate");
+ return queued;
+}
+
+/*
+ * /sys/fs/btrfs/<uuid>/stripe_reclaim_now: reclaim stripe_alloc
+ * raid56 block groups now, e.g. from an idle-time job, rather than waiting
+ * for the commit-time rescan and its five-minute hysteresis. The threshold
+ * is the space_info's bg_reclaim_threshold, or BTRFS_DEFAULT_RECLAIM_THRESH
+ * where that is 0 (reclaim off), since the request is explicit.
+ *
+ * @all: queue every such group the reclaim predicate picks at that
+ * threshold (relocate below it, compact above it once stripe_compact percent
+ * is trapped), ignoring the hysteresis.
+ *
+ * Otherwise @start names one group (its logical start): it is reclaimed
+ * whatever the trapped minimums say, compacted if it is above the threshold
+ * and has trapped stripes, relocated otherwise. -ENOENT: no group starts
+ * there; -EINVAL: not a stripe_alloc raid56 group (relocate
+ * any other group with balance's vrange filter); -EBUSY: the group is
+ * read-only (balance or scrub hold it); -EAGAIN: it is already on a list
+ * (queued, unused or new); -ENOSPC: the other groups' claimable stripes plus
+ * one new chunk cannot take what would move.
+ *
+ * Both return once the groups are queued, without waiting for the reclaim.
+ * The worker takes the balance exclusive operation and gives the list back
+ * when another operation holds it, so a request made while a balance,
+ * replace, resize or device add/remove runs fails at once with -EAGAIN
+ * rather than wait unseen; nothing is queued. The reclaim worker's own hold
+ * (a balance with no balance control) is not refused: it picks up the new
+ * entries.
+ */
+int btrfs_stripe_reclaim_now(struct btrfs_fs_info *fs_info, bool all, u64 start)
+{
+ struct btrfs_space_info *sinfo;
+ struct btrfs_block_group *bg;
+ enum btrfs_exclusive_operation op;
+ enum stripe_reclaim_kind kind;
+ int queued = 0;
+ int ret = 0;
+
+ if (sb_rdonly(fs_info->sb) || BTRFS_FS_ERROR(fs_info))
+ return -EROFS;
+ spin_lock(&fs_info->super_lock);
+ op = fs_info->exclusive_operation;
+ spin_unlock(&fs_info->super_lock);
+ if (op != BTRFS_EXCLOP_NONE &&
+ (op != BTRFS_EXCLOP_BALANCE || READ_ONCE(fs_info->balance_ctl)))
+ return -EAGAIN;
+
+ if (!all) {
+ bg = btrfs_lookup_block_group(fs_info, start);
+ if (!bg)
+ return -ENOENT;
+ if (bg->start != start) {
+ ret = -ENOENT;
+ } else if (!btrfs_is_stripe_alloc_bg(bg)) {
+ ret = -EINVAL;
+ } else if (READ_ONCE(bg->ro)) {
+ ret = -EBUSY;
+ } else {
+ int thresh = READ_ONCE(bg->space_info->bg_reclaim_threshold);
+
+ kind = btrfs_stripe_bg_reclaim_now_kind(bg,
+ thresh > 0 ? thresh : BTRFS_DEFAULT_RECLAIM_THRESH,
+ true, stripe_reclaim_chunk_room(bg));
+ if (kind == STRIPE_RECLAIM_NONE)
+ ret = READ_ONCE(bg->ro) ? -EBUSY : -ENOSPC;
+ else if (stripe_reclaim_now_queue(bg, kind))
+ queued = 1;
+ else
+ ret = -EAGAIN;
+ }
+ btrfs_put_block_group(bg);
+ if (ret)
+ return ret;
+ goto kick;
+ }
+
+ list_for_each_entry(sinfo, &fs_info->space_info, list) {
+ int thresh = READ_ONCE(sinfo->bg_reclaim_threshold);
+ int raid;
+
+ if (thresh <= 0)
+ thresh = BTRFS_DEFAULT_RECLAIM_THRESH;
+ down_read(&sinfo->groups_sem);
+ for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) {
+ if (!(btrfs_raid_array[raid].bg_flag &
+ BTRFS_BLOCK_GROUP_RAID56_MASK))
+ continue;
+ list_for_each_entry(bg, &sinfo->block_groups[raid], list) {
+ if (!btrfs_is_stripe_alloc_bg(bg))
+ continue;
+ kind = btrfs_stripe_bg_reclaim_now_kind(bg, thresh,
+ false, stripe_reclaim_chunk_room(bg));
+ if (kind != STRIPE_RECLAIM_NONE &&
+ stripe_reclaim_now_queue(bg, kind))
+ queued++;
+ }
+ }
+ up_read(&sinfo->groups_sem);
+ }
+ btrfs_debug(fs_info, "reclaim: stripe_reclaim_now queued %d block groups",
+ queued);
+kick:
+ if (queued)
+ queue_work(system_dfl_wq, &fs_info->reclaim_bgs_work);
+ return 0;
+}
+
static int read_bg_from_eb(struct btrfs_fs_info *fs_info, const struct btrfs_key *key,
const struct btrfs_path *path)
{
* and queueing a group with nowhere to move it only costs writers read-only
* transitions. Counters are read unlocked; the worker re-checks.
*/
-enum stripe_reclaim_kind {
- STRIPE_RECLAIM_NONE,
- STRIPE_RECLAIM_RELOCATE,
- STRIPE_RECLAIM_COMPACT,
-};
-
-static enum stripe_reclaim_kind stripe_bg_reclaim_kind(struct btrfs_block_group *bg,
- bool check_room)
+static enum stripe_reclaim_kind __stripe_bg_reclaim_kind(struct btrfs_block_group *bg,
+ int thresh, bool force,
+ bool check_room,
+ u64 extra_room)
{
struct btrfs_space_info *sinfo = bg->space_info;
- const int thresh = READ_ONCE(sinfo->bg_reclaim_threshold);
+ const bool ready = READ_ONCE(bg->stripe_unusable_ready);
const u64 used = READ_ONCE(bg->used);
- const u64 unusable = READ_ONCE(bg->stripe_unusable);
+ const u64 unusable = ready ? READ_ONCE(bg->stripe_unusable) : 0;
/*
* Stripes stranded behind live runs are trapped for admission but not
* for reclaim: they come back by themselves when the run is freed, and
u64 elsewhere;
u64 promised;
- if (thresh <= 0 || bg->ro || !READ_ONCE(bg->stripe_unusable_ready))
+ if (thresh <= 0 || bg->ro || (!ready && !force))
return STRIPE_RECLAIM_NONE;
- if (used >= mult_perc(bg->length, thresh)) {
+ if (force) {
+ /*
+ * Forced (stripe_reclaim_now naming this group): no trapped
+ * minimum. Compact a group too full to move whole if it has
+ * any trapped stripe, otherwise relocate it.
+ */
+ if (used >= mult_perc(bg->length, thresh) && trapped) {
+ kind = STRIPE_RECLAIM_COMPACT;
+ need = used / 2;
+ }
+ } else if (used >= mult_perc(bg->length, thresh)) {
/*
* Too full to move whole: compact once trapped space reaches
* stripe_compact percent of the group's length. (A share of
READ_ONCE(sinfo->bytes_stripe_margin);
elsewhere -= min(elsewhere, READ_ONCE(bg->stripe_claimable));
elsewhere -= min(elsewhere, promised);
- return elsewhere >= need ? kind : STRIPE_RECLAIM_NONE;
+ return elsewhere + extra_room >= need ? kind : STRIPE_RECLAIM_NONE;
+}
+
+static enum stripe_reclaim_kind stripe_bg_reclaim_kind(struct btrfs_block_group *bg,
+ bool check_room)
+{
+ return __stripe_bg_reclaim_kind(bg,
+ READ_ONCE(bg->space_info->bg_reclaim_threshold),
+ false, check_room, 0);
}
bool btrfs_stripe_bg_wants_reclaim(struct btrfs_block_group *bg)
return stripe_bg_reclaim_kind(bg, true) != STRIPE_RECLAIM_NONE;
}
+/*
+ * For stripe_reclaim_now: the reclaim @bg gets at threshold @thresh, room
+ * checked with @extra_room bytes (the chunk the worker may allocate) added to
+ * the other groups' claimable supply. @force drops the trapped minimums: see
+ * btrfs_stripe_reclaim_now().
+ */
+enum stripe_reclaim_kind btrfs_stripe_bg_reclaim_now_kind(struct btrfs_block_group *bg,
+ int thresh, bool force,
+ u64 extra_room)
+{
+ return __stripe_bg_reclaim_kind(bg, thresh, force, true, extra_room);
+}
+
/*
* For the reclaim worker, about a group it took: compact it rather than
* relocate it whole. Room is not re-checked; the worker makes room first,
BTRFS_ATTR_RW(, stripe_compact, btrfs_stripe_compact_show,
btrfs_stripe_compact_store);
+/*
+ * Reclaim stripe_alloc raid56 block groups now: "all" (or "1")
+ * queues every group the reclaim predicate picks, a block group's logical
+ * start forces that group. See btrfs_stripe_reclaim_now().
+ */
+static ssize_t btrfs_stripe_reclaim_now_store(struct kobject *kobj,
+ struct kobj_attribute *a,
+ const char *buf, size_t len)
+{
+ struct btrfs_fs_info *fs_info = to_fs_info(kobj);
+ u64 start = 0;
+ bool all;
+ int ret;
+
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+
+ all = sysfs_streq(buf, "all") || sysfs_streq(buf, "1");
+ if (!all) {
+ ret = kstrtou64(buf, 0, &start);
+ if (ret)
+ return ret;
+ }
+ ret = btrfs_stripe_reclaim_now(fs_info, all, start);
+ return ret ? ret : len;
+}
+BTRFS_ATTR_W(, stripe_reclaim_now, btrfs_stripe_reclaim_now_store);
+
static ssize_t btrfs_clone_alignment_show(struct kobject *kobj,
BTRFS_ATTR_PTR(, stripe_park_sync_timeout_ms),
BTRFS_ATTR_PTR(, stripe_park_congestion),
BTRFS_ATTR_PTR(, stripe_compact),
+ BTRFS_ATTR_PTR(, stripe_reclaim_now),
BTRFS_ATTR_PTR(, temp_fsid),
NULL,
};