u64 offset; /* next unallocated byte */
u64 inflight_bytes; /* reserved bytes with data IO pending */
u64 open_seq; /* fs_info->stripe_retire_seq at open */
- u64 owner; /* LOG class: btrfs_ino of the owner */
+ /*
+ * Private runs are found by (root, owner): root is the subvolume's
+ * root id, owner the inode number (LOG, NOCOW), a shared LOG key
+ * (bit 63 set, see stripe_run_owner()), or for META_LOG the root id
+ * again. Inode numbers are only unique within a subvolume.
+ */
+ u64 root;
+ u64 owner;
enum btrfs_stripe_run_class class;
bool open; /* accepting allocations */
bool persistent; /* survives commits; see close_block_group_stripe_runs() */
}
/*
- * Allocate num_bytes for a log-active inode from its private LOG-class run
- * in this block group, claiming a fresh run when the inode has none here or
- * the current one does not fit. Private runs are never placed in the shared
- * band slots and are found by owner lookup, so no other allocation can join
- * an inode's run and one inode's log commit settles only its own stripes.
+ * The owner a private run is found by: the inode, or with
+ * stripe_log_data_shared a LOG-class run shared by every inode of the
+ * subvolume logging in the same log transaction. An allocation of a full
+ * stripe or more keeps the inode's private run, so small fsync extents never
+ * share the partial last stripe of a large one and wait for its IO. Bit 63
+ * keeps the shared keys apart from inode numbers; the subvolume is matched
+ * separately, by run->root.
+ */
+static u64 stripe_run_owner(struct btrfs_inode *inode,
+ enum btrfs_stripe_run_class class, u64 num_bytes,
+ u64 fsl)
+{
+ struct btrfs_root *root = inode->root;
+
+ if (class == BTRFS_STRIPE_RUN_LOG && num_bytes < fsl &&
+ READ_ONCE(root->fs_info->stripe_log_data_shared))
+ return BIT_ULL(63) | (u32)READ_ONCE(root->log_transid);
+ return btrfs_ino(inode);
+}
+
+/*
+ * Allocate num_bytes for @inode from its private run of @class (LOG or
+ * NOCOW) in this block group, claiming a fresh run when there is none or it
+ * does not fit. With stripe_log_data_shared, small LOG allocations use the
+ * run the subvolume shares for the current log transaction
+ * (stripe_run_owner()). Private runs are never placed in a band slot and
+ * are found by subvolume and owner, so a log commit settles only stripes
+ * holding its own extents. A NOCOW allocation may instead re-adopt the
+ * free tail of the inode's last partial stripe (@readopt_hint).
*
- * A replaced run (too small for this allocation) is closed immediately: the
- * inode moves on to the fresh run, so keeping the remainder open would only
- * strand it until the commit-time retirement. Two racing allocations for
- * the same inode can each claim a fresh run; the owner walk then serves one
- * of them and the other drains and retires at the next commit -- an
- * occasional trapped tail, not a correctness problem.
+ * A run too small for this allocation is closed at once rather than left
+ * to strand its tail until commit. Two racing allocations for one owner
+ * may each claim a run; the spare drains at the next commit, costing a
+ * trapped tail, not correctness.
*
* Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in
* this block group can satisfy the allocation, or -ENOMEM.
{
struct btrfs_fs_info *fs_info = bg->fs_info;
const u64 fsl = bg->full_stripe_len;
- const u64 ino = btrfs_ino(inode);
+ const u64 root_id = btrfs_root_id(inode->root);
+ const u64 ino = stripe_run_owner(inode, class, num_bytes, fsl);
struct btrfs_open_stripe_run *new_run;
struct btrfs_open_stripe_run *run;
unsigned long flags;
spin_lock_irqsave(&bg->stripe_run_lock, flags);
list_for_each_entry(run, &bg->open_stripe_runs, list) {
- if (run->class != class || run->owner != ino || !run->open)
+ if (run->class != class || run->root != root_id ||
+ run->owner != ino || !run->open)
continue;
if (num_bytes <= run->end - run->offset) {
*ret_offset = run->offset;
* the re-adoption and falls through to a fresh claim below,
* costing this stripe's free tail, not correctness.
*/
- if (stripe_extents_owned_by(fs_info, btrfs_root_id(inode->root),
- ino, stripe, fsl, &frontier, true) &&
+ if (stripe_extents_owned_by(fs_info, root_id, ino, stripe, fsl,
+ &frontier, true) &&
frontier > stripe && frontier < stripe + fsl &&
num_bytes <= stripe + fsl - frontier) {
tlen = stripe + fsl - frontier;
BTRFS_STRIPE_RMW_PREALLOC));
if (class == BTRFS_STRIPE_RUN_NOCOW)
atomic64_inc(&fs_info->stripe_run_stats.nocow_run_opened);
+ new_run->root = root_id;
new_run->owner = ino;
new_run->start = start;
new_run->end = start + len;
if (cur < run->start || cur >= run->end)
continue;
if (run->class == BTRFS_STRIPE_RUN_NOCOW &&
- run->owner == ino)
+ run->root == root_id && run->owner == ino)
verdict = 1;
else
verdict = -1;
carry->range[carry->nr_ranges].len = file_len;
carry->range[carry->nr_ranges].disk_bytenr = bytenr;
carry->nr_ranges++;
+ atomic64_inc(&inode->root->fs_info->stripe_run_stats.log_carry_ranges);
}
/*
}
/*
- * Close every private stripe run owned by @inode, of any class: called at
- * inode eviction so a persistent nocow run's claimed tail does not stay
- * unusable for the rest of the mount (log-class runs would close at the
- * next commit anyway; closing them here is merely tidy). The runs' tails
- * return to the free space cache.
+ * Close @inode's open LOG and NOCOW runs at eviction, so a persistent NOCOW
+ * run's claimed tail does not stay unusable for the rest of the mount. The
+ * tails return to the free space cache.
*/
void btrfs_close_inode_stripe_runs(struct btrfs_inode *inode)
{
struct btrfs_fs_info *fs_info = inode->root->fs_info;
+ const u64 root_id = btrfs_root_id(inode->root);
const u64 ino = btrfs_ino(inode);
struct btrfs_block_group *bg;
spin_lock_irqsave(&bg->stripe_run_lock, flags);
list_for_each_entry(run, &bg->open_stripe_runs, list) {
- if (!run->open || run->owner != ino)
+ if (!run->open || run->root != root_id ||
+ run->owner != ino ||
+ (run->class != BTRFS_STRIPE_RUN_LOG &&
+ run->class != BTRFS_STRIPE_RUN_NOCOW))
continue;
tail_len = close_open_stripe_run(bg, run, &tail_start);
break;
spin_unlock(&fs_info->open_stripe_lock);
}
+/*
+ * A log commit on a shared LOG run: instead of closing the run, move its
+ * frontier past the stripe holding the end of the logged extent, so nothing
+ * allocates beside the extent after the log commit while the run goes on in
+ * the next stripe. The skipped sectors are not live (the pad oracle fills
+ * them) and go back to the free space cache like a closed run's tail: inside
+ * a partly live stripe they count as trapped until the stripe frees, and a
+ * later close returns only [offset, end), which no longer includes them.
+ * Caller holds bg->stripe_run_lock. Returns the skipped length.
+ */
+static u64 seal_log_stripe(struct btrfs_block_group *bg,
+ struct btrfs_open_stripe_run *run, u64 ext_end,
+ u64 *skip_start)
+{
+ const u64 fsl = bg->full_stripe_len;
+ u64 boundary;
+ u64 skip;
+
+ if (!run->open)
+ return 0;
+ boundary = bg->start + stripe_round_down(ext_end - 1 - bg->start, fsl) + fsl;
+ if (run->offset >= boundary)
+ return 0;
+ boundary = min(boundary, run->end);
+ skip = boundary - run->offset;
+ *skip_start = run->offset;
+ run->offset = boundary;
+ bg->stripe_open_remainder -= skip;
+ if (run->offset == run->end) {
+ int class;
+ int band;
+
+ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
+ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+ if (bg->open_stripe[class][band] == run)
+ bg->open_stripe[class][band] = NULL;
+ run->open = false;
+ if (open_stripe_run_drained(run) && !run->retained)
+ free_open_stripe_run(bg, run);
+ }
+ return skip;
+}
+
+static bool run_unwritten_in(struct btrfs_open_stripe_run *run, u64 start,
+ u64 len);
+
+/*
+ * Per-stripe settle (stripe_log_data_shared): every allocated sector of the
+ * stripes holding [bytenr, bytenr + num_bytes) has been written, or no run
+ * covers them any more. Writers sharing those stripes park and merge into
+ * the same full-stripe write, so this waits for them and for nothing else in
+ * the run.
+ */
+static bool stripe_log_extent_written(struct btrfs_block_group *bg,
+ u64 bytenr, u64 num_bytes)
+{
+ const u64 fsl = bg->full_stripe_len;
+ const u64 s = bg->start + stripe_round_down(bytenr - bg->start, fsl);
+ const u64 e = bg->start +
+ stripe_round_down(bytenr + num_bytes - bg->start + fsl - 1, fsl);
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ bool written = true;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (bytenr < run->start || bytenr >= run->offset)
+ continue;
+ if (run_unwritten_in(run, s, e - s))
+ written = false;
+ break;
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ return written;
+}
+
/*
* A log commit is about to make the extent at [bytenr, bytenr + num_bytes)
- * durable. Once the log super lands, no later write may extend or RMW the
- * extent's stripes -- the log-window analogue of invariant I2, which the
- * commit-time retirement provides for full commits. Close the open run
- * covering the extent so nothing further allocates into its stripes, kick
- * any parked partial writes for it, and wait for all data IO into it to
- * finish. Extents in full or already-settled stripes cost one list walk.
+ * durable, after which no later write may extend or RMW its stripes. Close
+ * the run covering the extent so nothing more allocates into its stripes,
+ * kick any parked partial writes there, and wait for all IO into the run to
+ * finish. This is a pure data-IO wait (no transaction joins), bounded by
+ * bio flight time because parked writes are flushed first, and is safe
+ * under the inode log mutex and a running transaction handle.
*
- * The drain is bounded: run inflight is counted from allocation, which
- * happens during writeback with submission following in the same pass, so
- * the wait is bio flight time plus the parked-write deadline that the flush
- * below short-circuits. It is a pure data-IO wait (no transaction joins),
- * safe under the inode log mutex and a running transaction handle.
+ * With stripe_log_data_shared a shared LOG run is not closed: the stripe
+ * holding the end of the extent is sealed (seal_log_stripe()). In that
+ * mode the wait, for any run, is for every allocated sector of the
+ * extent's stripes to be written (stripe_log_extent_written()) rather than
+ * for the run to drain. Neither wait has a timeout: returning early would
+ * let a neighbour's partial write RMW the extent after it is durable.
*
* [file_start, file_start + file_len) is the extent's file range, recorded
- * for carry-forward when the extent sits in the run's final partial stripe;
- * file_len == 0 opts the extent out of carrying (settling still happens).
+ * for carry-forward when the extent is in the run's final partial stripe
+ * (stripe_log_data_shared clear only); file_len == 0 opts out of carrying.
*/
void btrfs_log_settle_stripes(struct btrfs_inode *inode, u64 file_start,
u64 file_len, u64 bytenr, u64 num_bytes)
{
struct btrfs_fs_info *fs_info = inode->root->fs_info;
+ /* Once: the seal and the wait below must agree on the mode. */
+ const bool shared = READ_ONCE(fs_info->stripe_log_data_shared);
struct btrfs_block_group *bg;
struct btrfs_open_stripe_run *run;
unsigned long flags;
u64 flush_len = 0;
u64 ps_start = 0;
u64 ps_len = 0;
+ u64 seal_start = 0;
+ u64 seal_len = 0;
/* stripe_alloc_allow_rmw=fsync waives the log-commit guarantee. */
if (btrfs_stripe_allow_rmw(fs_info) & BTRFS_STRIPE_RMW_FSYNC)
*/
if (run->class == BTRFS_STRIPE_RUN_NOCOW)
break;
+ if (shared && (run->owner & BIT_ULL(63))) {
+ const u64 fsl = bg->full_stripe_len;
+
+ /* only the logged extent's stripes, not the shared run */
+ flush_start = bg->start + stripe_round_down(bytenr - bg->start, fsl);
+ flush_len = bg->start + stripe_round_down(bytenr + num_bytes - 1 - bg->start, fsl) +
+ fsl - flush_start;
+ seal_len = seal_log_stripe(bg, run, bytenr + num_bytes, &seal_start);
+ break;
+ }
flush_start = run->start;
flush_len = run->end - run->start;
/*
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
- log_carry_record(inode, file_start, file_len, bytenr, num_bytes,
- ps_start, ps_len);
+ if (!shared)
+ log_carry_record(inode, file_start, file_len, bytenr, num_bytes,
+ ps_start, ps_len);
+ if (seal_len) {
+ btrfs_add_free_space(bg, seal_start, seal_len);
+ stripe_open_remainder_sync(bg);
+ atomic64_inc(&fs_info->stripe_run_stats.log_seals);
+ atomic64_add(seal_len, &fs_info->stripe_run_stats.log_seal_bytes);
+ }
if (tail_len) {
btrfs_add_free_space(bg, tail_start, tail_len);
stripe_open_remainder_sync(bg);
}
if (flush_len) {
+ ktime_t t0;
+
btrfs_flush_parked_rbios(fs_info, flush_start, flush_len, false);
- wait_var_event(&bg->open_stripe_runs,
- stripe_log_range_settled(bg, bytenr) ||
- BTRFS_FS_ERROR(fs_info));
+ t0 = ktime_get();
+ if (shared)
+ wait_var_event(&bg->open_stripe_runs,
+ stripe_log_extent_written(bg, bytenr, num_bytes) ||
+ BTRFS_FS_ERROR(fs_info));
+ else
+ wait_var_event(&bg->open_stripe_runs,
+ stripe_log_range_settled(bg, bytenr) ||
+ BTRFS_FS_ERROR(fs_info));
+ atomic64_inc(&fs_info->stripe_run_stats.log_settle_waits);
+ atomic64_add(ktime_to_ns(ktime_sub(ktime_get(), t0)),
+ &fs_info->stripe_run_stats.log_settle_wait_ns);
}
btrfs_put_block_group(bg);
}
bitmap_set(run->written, first, nr);
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ /* Per-stripe log settles wait on the written map, not on a drain. */
+ wake_up_var(&bg->open_stripe_runs);
btrfs_put_block_group(bg);
if (not_live)
atomic64_add(not_live, &fs_info->stripe_park_stats.written_not_live);