u64 bytes;
int error;
bool steal;
+ /*
+ * A preallocation: one whole stripe of collateral is charged into
+ * bytes_stripe_margin in the same critical section that grants the
+ * bytes (stripe_hold_locked()), and handed back through @held.
+ */
+ bool hold_stripe;
+ u64 held;
+ /*
+ * The stripe_alloc collateral inside @bytes (the delalloc probe
+ * margin), which stripe_claimable_admit() leaves out of the charge.
+ */
+ u64 stripe_margin;
struct list_head list;
wait_queue_head_t wait;
spinlock_t lock;
if (!unit || !sinfo || mod == 0)
return;
+ /*
+ * Relocation's extents are already placed; its reservations are
+ * charged a margin of their own in stripe_claimable_admit().
+ */
+ if (btrfs_is_data_reloc_root(inode->root))
+ return;
if (btrfs_test_opt(fs_info, STRIPE_ALLOC)) {
/* charges and releases both live */
} else if (mod > 0 || READ_ONCE(sinfo->bytes_stripe_margin) == 0) {
spin_unlock(&sinfo->lock);
}
+/*
+ * Charge one whole stripe of margin for a preallocation, under
+ * space_info->lock in the same critical section that admits its bytes. A
+ * preallocation has no outstanding extent to carry a margin, yet its
+ * allocation claims its bytes rounded up to whole stripes. The hold keeps
+ * that stripe out of the supply other writers are admitted against until
+ * btrfs_stripe_prealloc_release(). Charged after admission, with the lock
+ * dropped in between, two preallocations could share one stripe of supply.
+ * Returns the bytes held, or 0 when stripe_alloc does not apply.
+ */
+static u64 stripe_hold_locked(struct btrfs_space_info *space_info)
+{
+ struct btrfs_fs_info *fs_info = space_info->fs_info;
+ const u64 unit = READ_ONCE(fs_info->stripe_margin_unit);
+
+ lockdep_assert_held(&space_info->lock);
+ if (!unit || !(space_info->flags & BTRFS_BLOCK_GROUP_DATA) ||
+ !btrfs_test_opt(fs_info, STRIPE_ALLOC))
+ return 0;
+ space_info->bytes_stripe_margin += unit;
+ return unit;
+}
+
+void btrfs_stripe_prealloc_release(struct btrfs_fs_info *fs_info, u64 held)
+{
+ struct btrfs_space_info *sinfo = fs_info->data_sinfo;
+
+ if (!held || !sinfo)
+ return;
+ spin_lock(&sinfo->lock);
+ sinfo->bytes_stripe_margin -= min(held, sinfo->bytes_stripe_margin);
+ btrfs_try_granting_tickets(sinfo);
+ spin_unlock(&sinfo->lock);
+}
+
struct btrfs_space_info *btrfs_find_space_info(const struct btrfs_fs_info *info,
u64 flags)
spin_unlock(&ticket->lock);
}
+/*
+ * The most whole stripes that @bytes of data can claim when writeback
+ * allocates them: in a group of full stripe f, an allocation claims
+ * roundup(bytes, f), so the most over every width a data group has. The
+ * widths are not powers of two for most geometries (3 data stripes: 192K),
+ * so divide. If there were more widths than fs_info tracks, fall back to
+ * bytes + unit - sectorsize, which bounds roundup(bytes, f) for every
+ * f <= unit (all sector multiples).
+ */
+static u64 stripe_whole_need(struct btrfs_fs_info *fs_info, u64 bytes, u64 unit)
+{
+ const int nr = smp_load_acquire(&fs_info->nr_stripe_widths);
+ u64 need = 0;
+
+ if (nr > BTRFS_STRIPE_WIDTHS_MAX)
+ return bytes + unit - fs_info->sectorsize;
+ for (int i = 0; i < nr; i++) {
+ const u64 w = READ_ONCE(fs_info->stripe_widths[i]);
+
+ need = max(need, div64_u64(bytes + w - 1, w) * w);
+ }
+ /* stripe_margin_unit is set first: no width recorded yet */
+ if (!need)
+ need = div64_u64(bytes + unit - 1, unit) * unit;
+ return need;
+}
+
+/*
+ * stripe_alloc data admission gate. A raid56 stripe_alloc data write can
+ * only be placed crash-safely in a fully free stripe, so admit a data
+ * reservation only while the claimable whole-stripe supply covers all
+ * outstanding reservations, their margins and this one. This turns a write
+ * that could not be placed into ENOSPC at write() rather than dropped data
+ * at writeback. It is pessimistic: open runs' remainders are not counted,
+ * and the commit rescan and FLUSH_DATA ticket retry recover that space.
+ * Space infos without raid56 data are unaffected (stripe_margin_unit is
+ * 0). Caller holds space_info->lock.
+ *
+ * @margin is the stripe_alloc collateral inside @bytes (the delalloc probe
+ * margin). It is not charged: once granted it is accounted in
+ * bytes_may_use, then in bytes_stripe_margin, where every later admission
+ * sees it.
+ */
+static bool stripe_claimable_admit(struct btrfs_space_info *space_info, u64 bytes,
+ u64 margin, bool reloc)
+{
+ struct btrfs_fs_info *fs_info = space_info->fs_info;
+ const u64 unit = READ_ONCE(fs_info->stripe_margin_unit);
+
+ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC) ||
+ !(space_info->flags & BTRFS_BLOCK_GROUP_DATA) || !unit)
+ return true;
+ /*
+ * Relocation carries no per-extent margin (its extents are already
+ * placed): charge it whole stripes plus one, the tail its own open
+ * runs can lose at the next commit.
+ */
+ {
+ u64 supply = space_info->bytes_stripe_claimable;
+ const u64 dedicated = space_info->bytes_stripe_claimable_reloc;
+
+ if (reloc) {
+ /*
+ * Against the whole supply, dedicated group included:
+ * when its group runs dry, relocation moves on to
+ * another. Whole stripes of the widest group; unit is
+ * not a power of two for most geometries, so divide.
+ */
+ bytes = div64_u64(bytes + unit - 1, unit) * unit + unit;
+ } else {
+ /* ...and that group is out of reach for everyone else */
+ supply -= min(supply, dedicated);
+ /*
+ * Charge what writeback can claim for the data: whole
+ * stripes of whichever group it lands in. Not the
+ * collateral: the data plus its collateral (the probe
+ * margin, then one unit in bytes_stripe_margin, or a
+ * preallocation's hold) stays accounted until the
+ * extent is written, and covers that claim, so the
+ * accounted total of every outstanding extent bounds
+ * what they can still claim. Charging the collateral
+ * as well would ask the supply for it twice.
+ */
+ bytes = stripe_whole_need(fs_info, bytes - min(bytes, margin),
+ unit);
+ }
+ /*
+ * Outstanding data extents hold their margin
+ * (bytes_stripe_margin) against the commit that traps their
+ * runs' tails. btrfs_space_info_used() counts it against
+ * total_bytes, but it is owed out of the whole-stripe supply,
+ * so every later admission, relocation included, must leave it
+ * there.
+ */
+ bytes += space_info->bytes_stripe_margin;
+ return space_info->bytes_may_use + bytes <= supply;
+ }
+}
+
/*
* This is for space we already have accounted in space_info->bytes_may_use, so
* basically when we're returning space from block_rsv's.
used_after = used + ticket->bytes;
/* Check and see if our ticket can be satisfied now. */
- if (used_after <= space_info->total_bytes ||
- can_overcommit(space_info, used, ticket->bytes, flush)) {
+ if ((used_after <= space_info->total_bytes ||
+ can_overcommit(space_info, used, ticket->bytes, flush)) &&
+ stripe_claimable_admit(space_info, ticket->bytes,
+ ticket->stripe_margin, false)) {
btrfs_space_info_update_bytes_may_use(space_info, ticket->bytes);
+ if (ticket->hold_stripe)
+ ticket->held = stripe_hold_locked(space_info);
remove_ticket(space_info, ticket, 0);
space_info->tickets_id++;
used = used_after;
switch (flush) {
case BTRFS_RESERVE_FLUSH_DATA:
+ case BTRFS_RESERVE_FLUSH_DATA_RELOC:
case BTRFS_RESERVE_FLUSH_ALL:
case BTRFS_RESERVE_FLUSH_ALL_STEAL:
wait_reserve_ticket(space_info, ticket);
*/
static inline bool can_ticket(enum btrfs_reserve_flush_enum flush)
{
+ /*
+ * Relocation never waits for space: at the fill edge the stripes it
+ * would wait for are the ones only it can free (stripe_alloc).
+ */
return (flush != BTRFS_RESERVE_NO_FLUSH &&
- flush != BTRFS_RESERVE_FLUSH_EMERGENCY);
+ flush != BTRFS_RESERVE_FLUSH_EMERGENCY &&
+ flush != BTRFS_RESERVE_FLUSH_DATA_RELOC);
}
/*
* regain reservations will be made and this will fail if there is not enough
* space already.
*/
-static int reserve_bytes(struct btrfs_space_info *space_info, u64 orig_bytes,
- enum btrfs_reserve_flush_enum flush)
+static int __reserve_bytes(struct btrfs_space_info *space_info, u64 orig_bytes,
+ u64 stripe_margin,
+ enum btrfs_reserve_flush_enum flush, u64 *held)
{
struct btrfs_fs_info *fs_info = space_info->fs_info;
struct work_struct *async_work;
ASSERT(flush != BTRFS_RESERVE_FLUSH_EVICT, "flush=%d", flush);
}
- if (flush == BTRFS_RESERVE_FLUSH_DATA)
+ if (flush == BTRFS_RESERVE_FLUSH_DATA ||
+ flush == BTRFS_RESERVE_FLUSH_DATA_RELOC)
async_work = &fs_info->async_data_reclaim_work;
else
async_work = &fs_info->async_reclaim_work;
*/
if (!pending_tickets &&
((used + orig_bytes <= space_info->total_bytes) ||
- can_overcommit(space_info, used, orig_bytes, flush))) {
+ can_overcommit(space_info, used, orig_bytes, flush)) &&
+ stripe_claimable_admit(space_info, orig_bytes, stripe_margin,
+ flush == BTRFS_RESERVE_FLUSH_DATA_RELOC)) {
btrfs_space_info_update_bytes_may_use(space_info, orig_bytes);
+ if (held)
+ *held = stripe_hold_locked(space_info);
ret = 0;
}
init_waitqueue_head(&ticket.wait);
spin_lock_init(&ticket.lock);
ticket.steal = can_steal(flush);
+ ticket.hold_stripe = held != NULL;
+ ticket.held = 0;
+ ticket.stripe_margin = stripe_margin;
if (trace_btrfs_reserve_ticket_enabled())
start_ns = ktime_get_ns();
if (!ret || !can_ticket(flush))
return ret;
- return handle_reserve_ticket(space_info, &ticket, start_ns, orig_bytes, flush);
+ ret = handle_reserve_ticket(space_info, &ticket, start_ns, orig_bytes, flush);
+ if (!ret && held)
+ *held = ticket.held;
+ return ret;
}
/*
{
int ret;
- ret = reserve_bytes(space_info, orig_bytes, flush);
+ ret = __reserve_bytes(space_info, orig_bytes, 0, flush, NULL);
if (ret == -ENOSPC) {
struct btrfs_fs_info *fs_info = space_info->fs_info;
return ret;
}
+/*
+ * Relocation never waits for data space (see can_ticket()), so its
+ * reservation never reaches the flush states that allocate a data chunk.
+ * When it does not fit in the chunks already there, allocate one and let the
+ * caller retry. Returns true if a chunk was allocated.
+ */
+static bool reloc_alloc_data_chunk(struct btrfs_space_info *space_info)
+{
+ struct btrfs_fs_info *fs_info = space_info->fs_info;
+ struct btrfs_trans_handle *trans;
+ int ret;
+
+ trans = btrfs_join_transaction(fs_info->tree_root);
+ if (IS_ERR(trans))
+ return false;
+ ret = btrfs_chunk_alloc(trans, space_info,
+ btrfs_get_alloc_profile(fs_info, space_info->flags),
+ CHUNK_ALLOC_FORCE);
+ btrfs_end_transaction(trans);
+ return ret > 0;
+}
+
/*
* Try to reserve data bytes for an allocation.
*
* This will reserve bytes from the data space info. If there is not enough
* space then we will attempt to flush space as specified by flush.
*/
+static int reserve_data_bytes(struct btrfs_space_info *space_info, u64 bytes,
+ u64 stripe_margin,
+ enum btrfs_reserve_flush_enum flush, u64 *held);
+
int btrfs_reserve_data_bytes(struct btrfs_space_info *space_info, u64 bytes,
+ u64 stripe_margin,
enum btrfs_reserve_flush_enum flush)
+{
+ return reserve_data_bytes(space_info, bytes, stripe_margin, flush, NULL);
+}
+
+/*
+ * Like btrfs_reserve_data_bytes(), for a preallocation: with @held the
+ * reservation also charges one whole stripe of stripe_alloc collateral,
+ * atomically with the bytes, and returns it for
+ * btrfs_stripe_prealloc_release() once the caller's allocations are done.
+ */
+int btrfs_reserve_data_bytes_held(struct btrfs_space_info *space_info, u64 bytes,
+ enum btrfs_reserve_flush_enum flush, u64 *held)
+{
+ return reserve_data_bytes(space_info, bytes, 0, flush, held);
+}
+
+static int reserve_data_bytes(struct btrfs_space_info *space_info, u64 bytes,
+ u64 stripe_margin,
+ enum btrfs_reserve_flush_enum flush, u64 *held)
{
struct btrfs_fs_info *fs_info = space_info->fs_info;
int ret;
ASSERT(flush == BTRFS_RESERVE_FLUSH_DATA ||
+ flush == BTRFS_RESERVE_FLUSH_DATA_RELOC ||
flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE ||
flush == BTRFS_RESERVE_FLUSH_ZONED_RELOCATION ||
flush == BTRFS_RESERVE_NO_FLUSH, "flush=%d", flush);
- ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_DATA,
+ ASSERT(!current->journal_info ||
+ (flush != BTRFS_RESERVE_FLUSH_DATA &&
+ flush != BTRFS_RESERVE_FLUSH_DATA_RELOC),
"current->journal_info=0x%lx flush=%d",
(unsigned long)current->journal_info, flush);
- ret = reserve_bytes(space_info, bytes, flush);
+ if (held)
+ *held = 0;
+ ret = __reserve_bytes(space_info, bytes, stripe_margin, flush, held);
+ /*
+ * Relocation does not wait for data space (can_ticket()), but must not
+ * fail while unallocated space remains, so allocate data chunks until
+ * the reservation fits. One chunk may not be enough: relocation
+ * reserves a whole cluster at once, and a new chunk can be smaller (the
+ * chunk size cap shrinks with the filesystem during a device remove).
+ * Each success consumes unallocated space, so the loop ends.
+ */
+ while (ret == -ENOSPC && flush == BTRFS_RESERVE_FLUSH_DATA_RELOC &&
+ reloc_alloc_data_chunk(space_info))
+ ret = __reserve_bytes(space_info, bytes, stripe_margin, flush,
+ held);
if (ret == -ENOSPC) {
trace_btrfs_space_reservation(fs_info, "space_info:enospc",
space_info->flags, bytes, 1);