u64 offset; /* next unallocated byte */
u64 inflight_bytes; /* reserved bytes with data IO pending */
u64 open_seq; /* fs_info->stripe_retire_seq at open */
+ u64 owner; /* LOG class: btrfs_ino of the owner */
enum btrfs_stripe_run_class class;
bool open; /* accepting allocations */
};
int band = stripe_run_band(bg, run->end - run->offset);
struct btrfs_open_stripe_run *occ = bg->open_stripe[class][band];
+ /* Per-inode LOG runs are private and never occupy a band slot. */
+ ASSERT(class != BTRFS_STRIPE_RUN_LOG);
+
if (occ) {
struct btrfs_open_stripe_run *keep, *shut;
}
new_run->bg = bg;
new_run->class = class;
+ new_run->owner = 0;
new_run->start = start;
new_run->end = start + len;
new_run->offset = start + num_bytes;
return ret;
}
+/*
+ * Allocate num_bytes for a log-active inode from its private LOG-class run
+ * in this block group, claiming a fresh run when the inode has none here or
+ * the current one does not fit. Private runs are never placed in the shared
+ * band slots and are found by owner lookup, so no other allocation can join
+ * an inode's run and one inode's log commit settles only its own stripes.
+ *
+ * A replaced run (too small for this allocation) is closed immediately: the
+ * inode moves on to the fresh run, so keeping the remainder open would only
+ * strand it until the commit-time retirement. Two racing allocations for
+ * the same inode can each claim a fresh run; the owner walk then serves one
+ * of them and the other drains and retires at the next commit -- an
+ * occasional trapped tail, not a correctness problem.
+ *
+ * Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in
+ * this block group can satisfy the allocation, or -ENOMEM.
+ */
+int btrfs_alloc_from_inode_stripe_run(struct btrfs_block_group *bg, u64 ino,
+ u64 num_bytes, u64 *ret_offset,
+ u64 *available)
+{
+ struct btrfs_fs_info *fs_info = bg->fs_info;
+ const u64 fsl = bg->full_stripe_len;
+ struct btrfs_open_stripe_run *new_run;
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ u64 tail_start = 0;
+ u64 tail_len = 0;
+ u64 start;
+ u64 len;
+ int ret;
+
+ ASSERT(num_bytes);
+ *available = 0;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (run->class != BTRFS_STRIPE_RUN_LOG || run->owner != ino ||
+ !run->open)
+ continue;
+ if (num_bytes <= run->end - run->offset) {
+ *ret_offset = run->offset;
+ run->offset += num_bytes;
+ run->inflight_bytes += num_bytes;
+ if (run->offset == run->end)
+ run->open = false; /* full; drains, is freed */
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ return 0;
+ }
+ tail_len = close_open_stripe_run(bg, run, &tail_start);
+ break;
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ if (tail_len)
+ btrfs_add_free_space(bg, tail_start, tail_len);
+
+ new_run = kmalloc(sizeof(*new_run), GFP_NOFS);
+ if (!new_run)
+ return -ENOMEM;
+
+ ret = btrfs_claim_free_stripe_run(bg,
+ div64_u64(num_bytes + fsl - 1, fsl) * fsl,
+ &start, &len);
+ if (ret)
+ goto out;
+ if (len < num_bytes) {
+ /* Only a shorter run is free; see btrfs_alloc_from_open_stripe(). */
+ btrfs_add_free_space(bg, start, len);
+ *available = len;
+ ret = -ENOSPC;
+ goto out;
+ }
+
+ /* See the installation comment in btrfs_alloc_from_open_stripe(). */
+ spin_lock(&fs_info->open_stripe_lock);
+ if (list_empty(&bg->open_stripe_bg_list)) {
+ btrfs_get_block_group(bg);
+ list_add_tail(&bg->open_stripe_bg_list,
+ &fs_info->open_stripe_bgs);
+ }
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ if (bg->ro) {
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ spin_unlock(&fs_info->open_stripe_lock);
+ btrfs_add_free_space(bg, start, len);
+ *available = 0;
+ ret = -ENOSPC;
+ goto out;
+ }
+ new_run->bg = bg;
+ new_run->class = BTRFS_STRIPE_RUN_LOG;
+ new_run->owner = ino;
+ new_run->start = start;
+ new_run->end = start + len;
+ new_run->offset = start + num_bytes;
+ new_run->inflight_bytes = num_bytes;
+ new_run->open_seq = fs_info->stripe_retire_seq;
+ new_run->open = (new_run->offset != new_run->end);
+ list_add_tail(&new_run->list, &bg->open_stripe_runs);
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ spin_unlock(&fs_info->open_stripe_lock);
+
+ *ret_offset = start;
+ new_run = NULL;
+ ret = 0;
+out:
+ kfree(new_run);
+ return ret;
+}
+
/*
* A live stripe run's address range must not be re-claimed while the run
* object still exists: an ENOSPC discard storm can return every byte of a
static void close_block_group_stripe_runs(struct btrfs_block_group *bg,
u64 seq)
{
- u64 tail_start[BTRFS_STRIPE_RUN_NR_CLASSES][BTRFS_STRIPE_RUN_NR_BANDS];
- u64 tail_len[BTRFS_STRIPE_RUN_NR_CLASSES][BTRFS_STRIPE_RUN_NR_BANDS];
struct btrfs_open_stripe_run *run;
unsigned long flags;
int class, band;
+restart:
spin_lock_irqsave(&bg->stripe_run_lock, flags);
- for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++) {
- for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++) {
- tail_len[class][band] = 0;
- run = bg->open_stripe[class][band];
- if (run && run->open_seq < seq) {
- bg->open_stripe[class][band] = NULL;
- tail_len[class][band] = close_open_stripe_run(
- bg, run, &tail_start[class][band]);
- }
- }
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ u64 tail_start;
+ u64 tail_len;
+
+ if (!run->open || run->open_seq >= seq)
+ continue;
+ for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
+ for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
+ if (bg->open_stripe[class][band] == run)
+ bg->open_stripe[class][band] = NULL;
+ tail_len = close_open_stripe_run(bg, run, &tail_start);
+ /*
+ * Closing may have freed the run, and returning its tail to
+ * the free space cache needs the lock dropped, so rescan.
+ * Each pass closes one run and open runs are bounded (band
+ * slots plus the private runs of inodes logged this window).
+ */
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ if (tail_len)
+ btrfs_add_free_space(bg, tail_start, tail_len);
+ goto restart;
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
- for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
- for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
- if (tail_len[class][band])
- btrfs_add_free_space(bg, tail_start[class][band],
- tail_len[class][band]);
}
/*
enum btrfs_stripe_run_class {
BTRFS_STRIPE_RUN_COW,
BTRFS_STRIPE_RUN_RELOC,
+ /*
+ * Private per-inode runs for log-active (fsync-heavy) inodes. Never
+ * placed in the shared band slots: each run is owned by one inode and
+ * found by owner lookup, so one inode's log commit settles only its
+ * own stripes.
+ */
+ BTRFS_STRIPE_RUN_LOG,
BTRFS_STRIPE_RUN_NR_CLASSES,
};
int btrfs_alloc_from_open_stripe(struct btrfs_block_group *bg, u64 num_bytes,
enum btrfs_stripe_run_class class,
u64 *ret_offset, u64 *available);
+int btrfs_alloc_from_inode_stripe_run(struct btrfs_block_group *bg, u64 ino,
+ u64 num_bytes, u64 *ret_offset,
+ u64 *available);
void btrfs_open_stripe_write_done(struct btrfs_block_group *bg, u64 start,
u64 num_bytes);
void btrfs_open_stripe_write_done_run(struct btrfs_open_stripe_run *run,
BTRFS_INODE_VERITY_IN_PROGRESS,
/* Set when this inode is a free space inode. */
BTRFS_INODE_FREE_SPACE_INODE,
+ /*
+ * Set the first time this inode is fsynced under stripe-exclusive
+ * allocation and never cleared while the inode stays cached. A
+ * log-active inode's datacow allocations are steered into the inode's
+ * own private LOG-class stripe run, so that its log commits settle
+ * only its own stripes instead of closing (and trapping the tails of)
+ * runs shared with other writers. Purely an allocation placement
+ * hint: correctness comes from the log-commit settling itself, which
+ * works on runs of any class.
+ */
+ BTRFS_INODE_LOG_ALLOC,
/* Set when there are no capabilities in XATTs for the inode. */
BTRFS_INODE_NO_CAP_XATTR,
/*
u64 ref_root_id;
};
+ /*
+ * Start of the last extent allocated from this inode's private
+ * LOG-class stripe run (see BTRFS_INODE_LOG_ALLOC). Advisory only:
+ * it seeds the allocator's search hint so consecutive allocations
+ * find the block group holding the inode's run. The run itself is
+ * found by owner lookup under the block group's stripe_run_lock, so a
+ * stale hint costs a miss, never a wrong run.
+ */
+ u64 log_run_hint;
+
/* Backwards incompatible flags, lower half of inode_item::flags */
u32 flags;
/* Read-only compatibility flags, upper half of inode_item::flags */
alloc_hint = btrfs_get_extent_allocation_hint(inode, start, len);
again:
- ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize,
+ ret = btrfs_reserve_extent(root, inode, len, len, fs_info->sectorsize,
0, alloc_hint, &ins, true, true);
if (ret == -EAGAIN) {
ASSERT(btrfs_is_zoned(fs_info));
if (skip)
return 1;
+ /*
+ * A log-active inode allocates from its own private LOG-class run so
+ * its log commits settle only its own stripes. If this block group
+ * has no fully-free stripes left for a private run, fall through to
+ * the shared runs: the data is then placed exactly as before 3b.1
+ * and the log-commit settling keeps it safe, just less cheaply.
+ */
+ if (ffe_ctl->for_log_inode) {
+ struct btrfs_inode *log_inode = ffe_ctl->for_log_inode;
+
+ ret = btrfs_alloc_from_inode_stripe_run(block_group,
+ btrfs_ino(log_inode), ffe_ctl->num_bytes,
+ &offset, &available);
+ if (ret == -ENOMEM)
+ return ret;
+ if (!ret) {
+ WRITE_ONCE(log_inode->log_run_hint, offset);
+ ffe_ctl->found_offset = offset;
+ ffe_ctl->search_start = offset;
+ return 0;
+ }
+ if (available > ffe_ctl->max_extent_size)
+ ffe_ctl->max_extent_size = available;
+ }
+
ret = btrfs_alloc_from_open_stripe(block_group, ffe_ctl->num_bytes,
class, &offset, &available);
if (ret == -ENOMEM)
* case -ENOSPC is returned then @ins->offset will contain the size of the
* largest available hole the allocator managed to find.
*/
-int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes,
- u64 num_bytes, u64 min_alloc_size,
+int btrfs_reserve_extent(struct btrfs_root *root, struct btrfs_inode *inode,
+ u64 ram_bytes, u64 num_bytes, u64 min_alloc_size,
u64 empty_size, u64 hint_byte,
struct btrfs_key *ins, bool is_data, bool delalloc)
{
int ret;
bool for_treelog = (btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID);
bool for_data_reloc = (btrfs_is_data_reloc_root(root) && is_data);
+ struct btrfs_inode *for_log_inode = NULL;
+
+ /*
+ * Steer a log-active inode's datacow data into its private LOG-class
+ * stripe run, seeded with the run's location as the search hint; see
+ * BTRFS_INODE_LOG_ALLOC. Purely placement: any allocation that
+ * cannot be served from a private run falls back to the shared runs.
+ */
+ if (inode && is_data && !for_data_reloc &&
+ btrfs_test_opt(fs_info, STRIPE_ALLOC) &&
+ test_bit(BTRFS_INODE_LOG_ALLOC, &inode->runtime_flags)) {
+ u64 log_hint = READ_ONCE(inode->log_run_hint);
+
+ for_log_inode = inode;
+ if (log_hint)
+ hint_byte = log_hint;
+ }
flags = get_alloc_profile_by_root(root, is_data);
again:
ffe_ctl.hint_byte = hint_byte;
ffe_ctl.for_treelog = for_treelog;
ffe_ctl.for_data_reloc = for_data_reloc;
+ ffe_ctl.for_log_inode = for_log_inode;
ret = find_free_extent(root, ins, &ffe_ctl);
if (!ret && !is_data) {
if (IS_ERR(block_rsv))
return ERR_CAST(block_rsv);
- ret = btrfs_reserve_extent(root, blocksize, blocksize, blocksize,
+ ret = btrfs_reserve_extent(root, NULL, blocksize, blocksize, blocksize,
empty_size, hint, &ins, false, false);
if (ret)
goto out_unuse;
/* Allocation is called for data relocation */
bool for_data_reloc;
+ /*
+ * Data allocation for a log-active inode under stripe-exclusive
+ * allocation: steer it into the inode's private LOG-class stripe run.
+ */
+ struct btrfs_inode *for_log_inode;
+
/*
* Set to true if we're retrying the allocation on this block group
* after waiting for caching progress, this is so that we retry only
int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
u64 root_objectid, u64 owner, u64 offset,
struct btrfs_key *ins);
-int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes, u64 num_bytes,
+int btrfs_reserve_extent(struct btrfs_root *root, struct btrfs_inode *inode,
+ u64 ram_bytes, u64 num_bytes,
u64 min_alloc_size, u64 empty_size, u64 hint_byte,
struct btrfs_key *ins, bool is_data, bool delalloc);
int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
trace_btrfs_sync_file_enter(file, datasync);
+ /*
+ * Under stripe-exclusive allocation, mark the inode log-active before
+ * this fsync flushes its delalloc, so that even the first fsync's
+ * allocations are steered into the inode's own LOG-class stripe run
+ * (see BTRFS_INODE_LOG_ALLOC). Sticky while the inode stays cached:
+ * one fsync is taken as a signal that more will follow.
+ */
+ if (btrfs_test_opt(fs_info, STRIPE_ALLOC) &&
+ !test_bit(BTRFS_INODE_LOG_ALLOC, &inode->runtime_flags))
+ set_bit(BTRFS_INODE_LOG_ALLOC, &inode->runtime_flags);
+
btrfs_init_log_ctx(&ctx, inode);
/*
}
compressed_size = async_extent->cb->bbio.bio.bi_iter.bi_size;
- ret = btrfs_reserve_extent(root, async_extent->ram_size,
+ ret = btrfs_reserve_extent(root, inode, async_extent->ram_size,
compressed_size, compressed_size,
0, *alloc_hint, &ins, true, true);
if (ret) {
u64 cur_end;
int ret;
- ret = btrfs_reserve_extent(root, num_bytes, num_bytes, min_alloc_size,
- 0, alloc_hint, ins, true, true);
+ ret = btrfs_reserve_extent(root, inode, num_bytes, num_bytes,
+ min_alloc_size, 0, alloc_hint, ins, true,
+ true);
if (ret < 0) {
*ret_alloc_size = cur_len;
return ret;
ei->last_unlink_trans = 0;
ei->last_reflink_trans = 0;
ei->last_log_commit = 0;
+ ei->log_run_hint = 0;
spin_lock_init(&ei->lock);
ei->outstanding_extents = 0;
* sized chunks.
*/
cur_bytes = min(cur_bytes, last_alloc);
- ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
+ ret = btrfs_reserve_extent(root, NULL, cur_bytes, cur_bytes,
min_size, 0, *alloc_hint, &ins, true, false);
if (ret)
break;
}
}
- ret = btrfs_reserve_extent(root, disk_num_bytes, disk_num_bytes,
+ ret = btrfs_reserve_extent(root, inode, disk_num_bytes, disk_num_bytes,
disk_num_bytes, 0, 0, &ins, true, true);
if (ret)
goto out_delalloc_release;
else
min_size = fs_info->nodesize;
- ret = btrfs_reserve_extent(fs_info->fs_root, length, length, min_size,
- 0, 0, &ins, is_data, false);
+ ret = btrfs_reserve_extent(fs_info->fs_root, NULL, length, length,
+ min_size, 0, 0, &ins, is_data, false);
if (unlikely(ret)) {
spin_lock(&sinfo->lock);
btrfs_space_info_update_bytes_may_use(sinfo, -length);
* of the identity remap that we're processing, and will tackle the
* rest of it the next time round.
*/
- ret = btrfs_reserve_extent(fs_info->fs_root, remap_length, remap_length,
- min_size, 0, 0, &ins, is_data, false);
+ ret = btrfs_reserve_extent(fs_info->fs_root, NULL, remap_length,
+ remap_length, min_size, 0, 0, &ins, is_data, false);
if (ret) {
spin_lock(&sinfo->lock);
btrfs_space_info_update_bytes_may_use(sinfo, -remap_length);
projects / linux / commitdiff