* runs left, or by btrfs_free_open_stripe_runs() at teardown.
*/
+static bool stripe_extents_owned_by(struct btrfs_fs_info *fs_info,
+ u64 root_id, u64 ino, u64 start, u64 len,
+ u64 *frontier, bool nowait);
+static bool stripe_range_run_free(struct btrfs_block_group *bg, u64 start,
+ u64 len);
+
struct btrfs_open_stripe_run {
struct list_head list; /* bg->open_stripe_runs */
struct btrfs_block_group *bg;
u64 owner; /* LOG class: btrfs_ino of the owner */
enum btrfs_stripe_run_class class;
bool open; /* accepting allocations */
+ bool persistent; /* survives commits; see close_block_group_stripe_runs() */
/*
* Relocation's run: kept after it closes and drains, until relocation
* releases it (btrfs_release_stripe_reloc_runs()). See
return !run->open && run->inflight_bytes == 0;
}
+/*
+ * Round to a full stripe.
+ *
+ * full_stripe_len is nr_data_stripes * stripe_len, which is not a power of
+ * two for most raid56 geometries (3 data stripes: 192K), so the mask-based
+ * round_down(), round_up() and IS_ALIGNED() must not be used. Divide
+ * instead.
+ */
+static u64 stripe_round_down(u64 val, u64 fsl)
+{
+ return div64_u64(val, fsl) * fsl;
+}
+
/*
* Band for a run with @remaining bytes of unallocated space: the power-of-two
* bucket of its remaining sectors (band k == remaining in [2^k, 2^(k+1))
bitmap_clear(run->live, first, nr);
}
+/*
+ * A run is dead when every byte it handed out has been freed again
+ * (btrfs_stripe_run_note_free() clears the live bits). It then protects no
+ * data, and the whole free stripes inside its range stay unclaimable
+ * (btrfs_stripe_run_range_usable()) until it is closed.
+ *
+ * Requires inflight_bytes == 0, since an in-flight write must keep its
+ * stripes, and a valid live map (live_nbits == 0 means liveness is
+ * unknown). Caller holds bg->stripe_run_lock.
+ */
+static bool open_stripe_run_dead(const struct btrfs_open_stripe_run *run)
+{
+ if (run->inflight_bytes)
+ return false;
+ if (run->offset <= run->start)
+ return false;
+ if (!run->live || !run->live_nbits)
+ return false;
+ return bitmap_empty(run->live, run->live_nbits);
+}
+
/* Free a drained run. Caller holds bg->stripe_run_lock. */
static void free_open_stripe_run(struct btrfs_block_group *bg,
struct btrfs_open_stripe_run *run)
}
new_run->bg = bg;
new_run->class = class;
+ new_run->persistent = false;
new_run->retained = class == BTRFS_STRIPE_RUN_RELOC;
if (new_run->retained)
atomic64_inc(&fs_info->stripe_run_stats.reloc_run_retained);
* Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in
* this block group can satisfy the allocation, or -ENOMEM.
*/
-int btrfs_alloc_from_inode_stripe_run(struct btrfs_block_group *bg, u64 ino,
+int btrfs_alloc_from_inode_stripe_run(struct btrfs_block_group *bg,
+ struct btrfs_inode *inode,
enum btrfs_stripe_run_class class,
- u64 num_bytes, u64 *ret_offset,
- u64 *available)
+ u64 num_bytes, u64 readopt_hint,
+ u64 *ret_offset, u64 *available)
{
struct btrfs_fs_info *fs_info = bg->fs_info;
const u64 fsl = bg->full_stripe_len;
+ const u64 ino = btrfs_ino(inode);
struct btrfs_open_stripe_run *new_run;
struct btrfs_open_stripe_run *run;
unsigned long flags;
if (!new_run)
return -ENOMEM;
+ /*
+ * Re-adoption (nocow only): after a remount (or eviction) the
+ * inode's last partial stripe has no run, and a fresh claim would
+ * abandon its free tail forever. When the allocation hint points
+ * into a stripe wholly owned by this inode, adopt the tail as the
+ * private run instead: every byte below the frontier is provably
+ * the inode's own committed data, and nothing else can consume a
+ * partial stripe's free space (see btrfs_claim_stripe_tail()).
+ */
+ if (class == BTRFS_STRIPE_RUN_NOCOW && readopt_hint &&
+ readopt_hint >= bg->start &&
+ readopt_hint < bg->start + bg->length) {
+ u64 stripe = bg->start +
+ stripe_round_down(readopt_hint - bg->start, fsl);
+ u64 frontier = 0;
+ u64 tlen;
+
+ /*
+ * Under space_info->groups_sem (we are inside
+ * find_free_extent()), so the lookup must not block -- see
+ * stripe_extents_owned_by(). A trylock failure just declines
+ * the re-adoption and falls through to a fresh claim below,
+ * costing this stripe's free tail, not correctness.
+ */
+ if (stripe_extents_owned_by(fs_info, btrfs_root_id(inode->root),
+ ino, stripe, fsl, &frontier, true) &&
+ frontier > stripe && frontier < stripe + fsl &&
+ num_bytes <= stripe + fsl - frontier) {
+ tlen = stripe + fsl - frontier;
+ /*
+ * btrfs_stripe_run_range_usable() is for whole-stripe
+ * claims (it rejects anything shorter than a full
+ * stripe); a tail needs only the run-overlap check.
+ */
+ if (stripe_range_run_free(bg, frontier, tlen) &&
+ !btrfs_claim_stripe_tail(bg, frontier, tlen)) {
+ start = frontier;
+ len = tlen;
+ goto install;
+ }
+ }
+ }
+
ret = btrfs_claim_free_stripe_run(bg,
div64_u64(num_bytes + fsl - 1, fsl) * fsl,
&start, &len);
goto out;
}
+install:
/* Sleeps, so it has to happen before the locks below. */
new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
new_run->written = new_run->live_nbits ?
}
new_run->bg = bg;
new_run->class = class;
+ /*
+ * A NOCOW run outlives commits only while the stripe_alloc_allow_rmw
+ * policy lets nocow data be written in place at all (see
+ * close_block_group_stripe_runs()). Decided once, here: the commit
+ * path's close and settle predicates must agree with each other for
+ * the run's whole life, whatever the policy does meanwhile.
+ */
+ new_run->persistent = class == BTRFS_STRIPE_RUN_NOCOW &&
+ (btrfs_stripe_allow_rmw(fs_info) &
+ (BTRFS_STRIPE_RMW_NODATACOW |
+ BTRFS_STRIPE_RMW_PREALLOC));
if (class == BTRFS_STRIPE_RUN_NOCOW)
atomic64_inc(&fs_info->stripe_run_stats.nocow_run_opened);
new_run->owner = ino;
return ret;
}
+/*
+ * An extent inside a stripe run's range has been freed: its sectors no longer
+ * hold data anyone intends to write, so clear their live bits.
+ *
+ * This is what makes a run detectably dead (open_stripe_run_dead()), so the
+ * next commit can close it and release its stripes. Without it a nodatacow
+ * or preallocating inode's private run hoards its stripes until the inode is
+ * evicted. A partial write to one of those stripes can also pad over the
+ * freed sectors now instead of falling back to a read-modify-write.
+ *
+ * Takes only bg->stripe_run_lock, and the callers hold no free space lock
+ * yet, so no nesting rule applies. The range is clamped to the run: a free
+ * may span a run's boundary, and running off the live map would retire it.
+ */
+void btrfs_stripe_run_note_free(struct btrfs_block_group *bg, u64 bytenr,
+ u64 size)
+{
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+
+ if (!btrfs_is_stripe_alloc_bg(bg))
+ return;
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ u64 start = max(bytenr, run->start);
+ u64 end = min(bytenr + size, run->offset);
+
+ /*
+ * Only the persistent runs, which are the only ones that can
+ * outlive their own data. Every other class is closed and
+ * freed by the commit it belongs to, so its live map is never
+ * consulted after its bytes could have been freed, and
+ * clearing bits there would change padding decisions for no
+ * gain here.
+ */
+ if (!run->persistent)
+ continue;
+ if (end <= start)
+ continue;
+ run_live_set(run, start, end - start, false);
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+}
+
/*
* A run's range must not be claimed again while the run object exists, even
* if every byte of one of its stripes is back in the free space cache
* @ino in @root_id -- exactly one reference, a plain data ref, count 1?
* Any metadata block, shared reference, or foreign owner fails. Caller
* limits the range to full stripes; uncommitted allocations need not be
- * visible because nothing can allocate into a partially used stripe (the
- * fully-free claim rule), so a foreign extent can only appear here after
- * the whole stripe frees and commits -- which the extent tree shows.
+ * visible because nothing can allocate into a partially used stripe, so a
+ * foreign extent can only appear after the whole stripe frees and commits.
+ *
+ * Callers holding space_info->groups_sem (anything reached from
+ * find_free_extent()) must pass @nowait: delayed ref processing takes
+ * extent tree locks and then groups_sem, so blocking on an extent tree lock
+ * here can deadlock behind a queued groups_sem writer. Callers that must
+ * not block for other reasons pass it too. With @nowait a contended lock
+ * fails the search and the answer is false, which every caller treats only
+ * as a reason to skip an optimisation.
*/
static bool stripe_extents_owned_by(struct btrfs_fs_info *fs_info,
- u64 root_id, u64 ino, u64 start, u64 len)
+ u64 root_id, u64 ino, u64 start, u64 len,
+ u64 *frontier, bool nowait)
{
struct btrfs_root *extent_root = btrfs_extent_root(fs_info, start);
struct btrfs_path *path;
path = btrfs_alloc_path();
if (!path)
return false;
+ /*
+ * Read-only search, so nowait is allowed, and every step of the walk
+ * below honours it (btrfs_next_leaf() runs with time_seq 0).
+ */
+ path->nowait = nowait;
key.objectid = start;
key.type = BTRFS_EXTENT_ITEM_KEY;
ret = false;
goto out;
}
+ if (frontier && key.objectid + key.offset > *frontier)
+ *frontier = min(key.objectid + key.offset, start + len);
path->slots[0]++;
}
out:
* overwrite passes.
*/
bool btrfs_stripe_nocow_writable(struct btrfs_inode *inode, u64 start,
- u64 num_bytes, u32 which)
+ u64 num_bytes, u32 which, bool nowait)
{
struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct btrfs_block_group *bg;
}
fsl = bg->full_stripe_len;
- cur = bg->start + round_down(start - bg->start, fsl);
+ cur = bg->start + stripe_round_down(start - bg->start, fsl);
end = start + num_bytes;
while (cur < end) {
struct btrfs_open_stripe_run *run;
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ /*
+ * No groups_sem here (this runs from the nocow check, not
+ * from the allocator), so the reliable blocking search is
+ * both safe and wanted: a spurious "not ours" would force
+ * COW on an extent that does not need it. A NOWAIT write
+ * must not block, so it gets the nonblocking search, and
+ * COW (which it cannot do either) when that cannot answer.
+ */
if (verdict == 0 &&
- !stripe_extents_owned_by(fs_info, root_id, ino, cur, fsl))
+ !stripe_extents_owned_by(fs_info, root_id, ino, cur, fsl,
+ NULL, nowait))
verdict = -1;
if (verdict < 0) {
ret = false;
}
+/*
+ * Disk end of @inode's last regular file extent, or 0: the re-adoption
+ * candidate for a nocow allocation arriving with no allocation hint --
+ * after a remount or eviction the extent maps are gone, so the normal
+ * em-based hint is empty exactly when re-adoption matters most.
+ */
+u64 btrfs_nocow_alloc_hint(struct btrfs_inode *inode)
+{
+ struct btrfs_root *root = inode->root;
+ struct btrfs_path *path;
+ struct btrfs_key key;
+ struct btrfs_file_extent_item *fi;
+ u64 hint = 0;
+ int ret;
+
+ path = btrfs_alloc_path();
+ if (!path)
+ return 0;
+ key.objectid = btrfs_ino(inode);
+ key.type = BTRFS_EXTENT_DATA_KEY;
+ key.offset = (u64)-1;
+ ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
+ if (ret < 0)
+ goto out;
+ ret = btrfs_previous_item(root, path, key.objectid,
+ BTRFS_EXTENT_DATA_KEY);
+ if (ret)
+ goto out;
+ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
+ fi = btrfs_item_ptr(path->nodes[0], path->slots[0],
+ struct btrfs_file_extent_item);
+ if (btrfs_file_extent_type(path->nodes[0], fi) ==
+ BTRFS_FILE_EXTENT_INLINE)
+ goto out;
+ if (btrfs_file_extent_disk_bytenr(path->nodes[0], fi))
+ hint = btrfs_file_extent_disk_bytenr(path->nodes[0], fi) +
+ btrfs_file_extent_disk_num_bytes(path->nodes[0], fi);
+out:
+ btrfs_free_path(path);
+ return hint;
+}
+
+/* Does no run object (open or draining) overlap [start, start + len)? */
+static bool stripe_range_run_free(struct btrfs_block_group *bg, u64 start,
+ u64 len)
+{
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ bool ret = true;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (run->start < start + len && run->end > start) {
+ ret = false;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ return ret;
+}
+
+/* Does any stripe run, open or draining, still cover part of @bg? */
+bool btrfs_block_group_has_stripe_runs(struct btrfs_block_group *bg)
+{
+ unsigned long flags;
+ bool ret;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ ret = !list_empty(&bg->open_stripe_runs);
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ return ret;
+}
+
+/*
+ * Close every private stripe run owned by @inode, of any class: called at
+ * inode eviction so a persistent nocow run's claimed tail does not stay
+ * unusable for the rest of the mount (log-class runs would close at the
+ * next commit anyway; closing them here is merely tidy). The runs' tails
+ * return to the free space cache.
+ */
+void btrfs_close_inode_stripe_runs(struct btrfs_inode *inode)
+{
+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
+ const u64 ino = btrfs_ino(inode);
+ struct btrfs_block_group *bg;
+
+ if (list_empty_careful(&fs_info->open_stripe_bgs))
+ return;
+restart:
+ spin_lock(&fs_info->open_stripe_lock);
+ list_for_each_entry(bg, &fs_info->open_stripe_bgs,
+ open_stripe_bg_list) {
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ u64 tail_start = 0;
+ u64 tail_len = 0;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (!run->open || run->owner != ino)
+ continue;
+ tail_len = close_open_stripe_run(bg, run, &tail_start);
+ break;
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ if (tail_len) {
+ btrfs_get_block_group(bg);
+ spin_unlock(&fs_info->open_stripe_lock);
+ btrfs_add_free_space(bg, tail_start, tail_len);
+ btrfs_put_block_group(bg);
+ goto restart;
+ }
+ }
+ spin_unlock(&fs_info->open_stripe_lock);
+}
+
/*
* Can stripe-exclusive allocation be enabled on this filesystem? Called for
* the "stripe_alloc" filesystem property, on a mounted (or mounting, options
static void close_block_group_stripe_runs(struct btrfs_block_group *bg,
u64 seq)
{
+ struct btrfs_fs_info *fs_info = bg->fs_info;
struct btrfs_open_stripe_run *run;
unsigned long flags;
int class, band;
if (!run->open || run->open_seq >= seq)
continue;
+ /*
+ * A persistent NOCOW run holds only its inode's
+ * write-hole-waived data, so commit retirement protects
+ * nothing there, and closing it each commit would waste a
+ * stripe per commit on a slowly appended file. It closes at
+ * forced quiescing (seq == U64_MAX), at the inode's eviction,
+ * or once it is dead.
+ */
+ if (run->persistent && seq != U64_MAX) {
+ /* A dead run protects nothing, only strands stripes. */
+ if (!open_stripe_run_dead(run))
+ continue;
+ atomic64_inc(&fs_info->stripe_run_stats.nocow_run_dead_closed);
+ }
for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
if (bg->open_stripe[class][band] == run)
spin_lock_irqsave(&bg->stripe_run_lock, flags);
list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ /* Persistent nocow runs are not part of any commit window. */
+ if (run->persistent && seq != U64_MAX)
+ continue;
/* Nor are retained relocation runs, once nothing is in flight. */
if (run->retained && open_stripe_run_drained(run) &&
seq != U64_MAX)
goto next;
}
+ /*
+ * A stripe run can still own space here that is neither used
+ * nor reserved: the free tail of a persistent nocow run, or
+ * the running transaction's metadata run. Its close would
+ * return that space to a removed group. The group goes back
+ * to read-write for the retry: a persistent run closes at the
+ * next commit once it is dead, a metadata run in
+ * btrfs_retire_meta_stripes(); try again after that commit.
+ */
+ if (btrfs_block_group_has_stripe_runs(block_group)) {
+ btrfs_dec_block_group_ro(block_group);
+ btrfs_link_bg_list(block_group, &retry_list);
+ goto next;
+ }
+
ret = btrfs_zone_finish(block_group);
if (ret < 0) {
btrfs_dec_block_group_ro(block_group);