* later allocation's write serializes behind the caller's stripe lock
* and overwrites the zeros.
*/
+/*
+ * How far has the covering run allocated into this stripe? Returns the
+ * exclusive end of the allocated prefix, clamped to the stripe, or U64_MAX
+ * when no run covers the stripe (nothing left to wait for; the caller
+ * proceeds as if the stripe were fully allocated and fully arrived).
+ * A partial write below the returned ceiling is waiting on neighbours whose
+ * data IO is already in flight -- allocation happens at writeback submission
+ * and the run's inflight accounting makes the commit wait for it -- so a
+ * parked rbio covering [stripe_start, ceiling) can complete (padding the
+ * dead tail), and one that does not should keep waiting for merges.
+ */
+u64 btrfs_stripe_run_alloc_ceiling(struct btrfs_fs_info *fs_info,
+ u64 stripe_start, u64 stripe_len)
+{
+ struct btrfs_block_group *bg;
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ u64 ceiling = U64_MAX;
+
+ if (list_empty_careful(&fs_info->open_stripe_bgs))
+ return U64_MAX;
+ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+ if (!bg)
+ return U64_MAX;
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (stripe_start < run->start || stripe_start >= run->end)
+ continue;
+ ceiling = clamp(run->offset, stripe_start,
+ stripe_start + stripe_len);
+ break;
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ btrfs_put_block_group(bg);
+ return ceiling;
+}
+
bool btrfs_stripe_run_pad_start(struct btrfs_fs_info *fs_info,
u64 stripe_start, u64 stripe_len,
u64 *pad_from,
stripe_open_remainder_sync(bg);
}
if (flush_len) {
- btrfs_flush_parked_rbios(fs_info, flush_start, flush_len);
+ btrfs_flush_parked_rbios(fs_info, flush_start, flush_len, false);
wait_var_event(&bg->open_stripe_runs,
stripe_log_range_settled(bg, bytenr));
}
{
close_block_group_stripe_runs(bg, U64_MAX);
/* Parked partial-stripe rbios hold the very bios the wait drains. */
- btrfs_flush_parked_rbios(bg->fs_info, bg->start, bg->length);
+ btrfs_flush_parked_rbios(bg->fs_info, bg->start, bg->length, false);
wait_var_event(&bg->open_stripe_runs,
bg_open_stripes_settled(bg, U64_MAX));
}
* A racing park that saw its run still open lands after this flush
* and is bounded by the park timer, not by us.
*/
- btrfs_flush_parked_rbios(fs_info, 0, U64_MAX);
+ btrfs_flush_parked_rbios(fs_info, 0, U64_MAX, false);
while (!list_empty(&retire_list)) {
bg = list_first_entry(&retire_list, struct btrfs_block_group,
/* Set at park time and never cleared: this rbio once parked (stats). */
#define RBIO_WAS_PARKED_BIT 7
+/*
+ * Set when a parked rbio carries (or merged) a REQ_SYNC bio: a waiter
+ * is blocked on it, so deadlines stay authoritative and neither the
+ * timer nor a flush may hold it back to wait for neighbours.
+ */
+#define RBIO_SYNC_PARK_BIT 8
+
#define RBIO_CACHE_SIZE 1024
/*
return false;
}
+/*
+ * A parked rbio is ready to go down when its gathered bios cover every
+ * byte the covering run has allocated in its stripe: the remainder lies
+ * at or past the frontier and the pad turns it into a full-stripe write.
+ * Anything less is waiting on an allocated neighbour whose data IO is
+ * already in flight, so waiting is bounded and reading (RMW) is waste.
+ * bio_list_bytes is read unlocked: merges only grow it, and a stale low
+ * value just delays readiness by one scan tick.
+ */
+static bool parked_rbio_ready(struct btrfs_raid_bio *rbio)
+{
+ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+ const u64 stripe_start = rbio->bioc->full_stripe_logical;
+ const u64 stripe_len = (u64)rbio->nr_data * BTRFS_STRIPE_LEN;
+ u64 ceiling;
+
+ ceiling = btrfs_stripe_run_alloc_ceiling(fs_info, stripe_start,
+ stripe_len);
+ if (ceiling == U64_MAX)
+ return true;
+ return READ_ONCE(rbio->bio_list_bytes) >= ceiling - stripe_start;
+}
+
/*
* Park a partial write rbio that owns its stripe lock, if its stripe is
* covered by an open stripe run. Returns true if the rbio was parked (or
spin_lock(&table->parked_lock);
set_bit(RBIO_PARKED_BIT, &rbio->flags);
set_bit(RBIO_WAS_PARKED_BIT, &rbio->flags);
+ if (timeout_ms == sync_timeout_ms)
+ set_bit(RBIO_SYNC_PARK_BIT, &rbio->flags);
rbio->park_deadline = jiffies + msecs_to_jiffies(timeout_ms);
+ rbio->park_stuck_deadline = jiffies +
+ msecs_to_jiffies(timeout_ms * 10);
list_add_tail(&rbio->parked_node, &table->parked);
spin_unlock(&table->parked_lock);
atomic64_inc(&fs_info->stripe_park_stats.parked);
* write. Callers may hold the stripe hash and bio_list locks; parked_lock
* nests inside both.
*/
-static void unpark_full_rbio(struct btrfs_raid_bio *rbio)
+static void unpark_ready_rbio(struct btrfs_raid_bio *rbio,
+ atomic64_t *counter)
{
struct btrfs_stripe_hash_table *table =
rbio->bioc->fs_info->stripe_hash_table;
}
rbio_unpark_locked(rbio);
spin_unlock(&table->parked_lock);
- atomic64_inc(&rbio->bioc->fs_info->stripe_park_stats.filled);
+ atomic64_inc(counter);
start_async_work(rbio, rmw_rbio_work_locked);
}
* ordered-extent waiters, and the park timer.
*/
void btrfs_flush_parked_rbios(struct btrfs_fs_info *fs_info, u64 start,
- u64 num_bytes)
+ u64 num_bytes, bool force)
{
struct btrfs_stripe_hash_table *table = fs_info->stripe_hash_table;
struct btrfs_raid_bio *rbio;
if (rbio_start + rbio_len <= start ||
(num_bytes != U64_MAX && rbio_start >= start + num_bytes))
continue;
+ /*
+ * A run-retirement flush precedes the commit's IO drain,
+ * which waits for the very arrivals this rbio is missing:
+ * leave it parked and the arrivals will complete it (the
+ * run is closed, so its allocated prefix is frozen). Only
+ * waiter-driven flushes (fsync, ordered-extent waits) force
+ * an incomplete stripe down, and sync parks always go.
+ */
+ if (!force && !test_bit(RBIO_SYNC_PARK_BIT, &rbio->flags) &&
+ !parked_rbio_ready(rbio))
+ continue;
rbio_unpark_locked(rbio);
list_add_tail(&rbio->parked_node, &flush);
}
rearm = true;
continue;
}
+ /*
+ * Past the deadline but still missing allocated
+ * neighbours: their data IO is in flight, so expiring
+ * now would read (RMW) what a short wait merges for
+ * free. Hold the park up to the stuck cap -- the cap
+ * only fires when an arrival never comes (writeback
+ * error, or a stripe already written once by a forced
+ * sync park).
+ */
+ if (!test_bit(RBIO_SYNC_PARK_BIT, &rbio->flags) &&
+ time_before(jiffies, rbio->park_stuck_deadline) &&
+ !parked_rbio_ready(rbio)) {
+ rearm = true;
+ continue;
+ }
rbio_unpark_locked(rbio);
list_add_tail(&rbio->parked_node, &flush);
}
spin_unlock(&table->parked_lock);
list_for_each_entry_safe(rbio, tmp, &flush, parked_node) {
+ struct btrfs_fs_info *ffs = rbio->bioc->fs_info;
+
list_del_init(&rbio->parked_node);
- atomic64_inc(&rbio->bioc->fs_info->stripe_park_stats.expired);
+ if (!test_bit(RBIO_SYNC_PARK_BIT, &rbio->flags) &&
+ time_after_eq(jiffies, rbio->park_stuck_deadline))
+ atomic64_inc(&ffs->stripe_park_stats.stuck);
+ else
+ atomic64_inc(&ffs->stripe_park_stats.expired);
start_async_work(rbio, rmw_rbio_work_locked);
}
if (rearm)
* this stripe.
*/
if (test_bit(RBIO_PARKED_BIT, &cur->flags)) {
+ struct btrfs_fs_info *cfs =
+ cur->bioc->fs_info;
+
if (cur->bio_list_bytes ==
cur->nr_data * BTRFS_STRIPE_LEN) {
- unpark_full_rbio(cur);
+ unpark_ready_rbio(cur,
+ &cfs->stripe_park_stats.filled);
+ } else if (parked_rbio_ready(cur)) {
+ /*
+ * This merge completed the allocated
+ * prefix; the rest of the stripe is
+ * dead space the pad fills.
+ */
+ unpark_ready_rbio(cur,
+ &cfs->stripe_park_stats.unparked_ready);
} else if (sync) {
struct btrfs_stripe_hash_table *table =
cur->bioc->fs_info->stripe_hash_table;
spin_lock(&table->parked_lock);
if (test_bit(RBIO_PARKED_BIT,
- &cur->flags) &&
- time_before(dl, cur->park_deadline))
- cur->park_deadline = dl;
+ &cur->flags)) {
+ set_bit(RBIO_SYNC_PARK_BIT,
+ &cur->flags);
+ if (time_before(dl,
+ cur->park_deadline))
+ cur->park_deadline = dl;
+ }
spin_unlock(&table->parked_lock);
}
}
projects / linux / commitdiff