return ret;
}
+/*
+ * Does @logical lie within a stripe run (open or draining)? Used by the
+ * raid56 layer to decide whether a partial write to this stripe may be
+ * parked to collect into a full-stripe write: an open run guarantees the
+ * rest of the stripe is either filled before the run closes or never, and
+ * a draining run's frozen frontier makes "never" decidable -- everything
+ * allocated below it is inflight IO the commit drain already waits for,
+ * and everything above it is dead space the pad fills. Excluding draining
+ * runs made every run's tail writes unparkable -- a run closes on
+ * exhaustion in the gap between allocation and write arrival -- and those
+ * writes are the bulk of stripe_alloc's read-modify-writes.
+ */
+bool btrfs_stripe_in_open_run(struct btrfs_fs_info *fs_info, u64 logical)
+{
+ struct btrfs_block_group *bg;
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ bool ret = false;
+
+ bg = btrfs_lookup_block_group(fs_info, logical);
+ if (!bg)
+ return false;
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (logical >= run->start && logical < run->end) {
+ ret = true;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ btrfs_put_block_group(bg);
+ return ret;
+}
+
/*
* Report completed (or abandoned) data IO for an allocation made with
* btrfs_alloc_from_open_stripe(). Every allocated byte must be reported
void btrfs_retire_block_group_stripes(struct btrfs_block_group *bg)
{
close_block_group_stripe_runs(bg, U64_MAX);
+ /* Parked partial-stripe rbios hold the very bios the wait drains. */
+ btrfs_flush_parked_rbios(bg->fs_info, bg->start, bg->length);
wait_var_event(&bg->open_stripe_runs,
bg_open_stripes_settled(bg, U64_MAX));
}
list_for_each_entry(bg, &retire_list, open_stripe_retire_list)
close_block_group_stripe_runs(bg, seq);
+ /*
+ * Parked partial-stripe write rbios (see raid56.c) hold bios whose
+ * write_done the drain below waits for; kick them all down first.
+ * A racing park that saw its run still open lands after this flush
+ * and is bounded by the park timer, not by us.
+ */
+ btrfs_flush_parked_rbios(fs_info, 0, U64_MAX);
+
while (!list_empty(&retire_list)) {
bg = list_first_entry(&retire_list, struct btrfs_block_group,
open_stripe_retire_list);
*/
#define RBIO_CACHE_READY_BIT 3
+/*
+ * Set while a partial write rbio is parked: it owns the stripe lock but its
+ * RMW has not been started, so writes filling the rest of an open stripe
+ * run's stripe merge into it (see rbio_try_park()). Whoever clears this bit
+ * owns starting the rbio's work, exactly once.
+ */
+#define RBIO_PARKED_BIT 4
+
#define RBIO_CACHE_SIZE 1024
+/*
+ * How long a parked partial write rbio may wait for merges before flushing.
+ * Sync writes (fsync- or sync-driven writeback, REQ_SYNC) have a waiter
+ * blocked on their completion, so they get a much shorter deadline: long
+ * enough for the rest of one writeback pass over the same stripe to merge
+ * in, short enough to stay invisible in fsync latency.
+ */
+#define BTRFS_RBIO_PARK_TIMEOUT_MS 100
+#define BTRFS_RBIO_PARK_SYNC_TIMEOUT_MS 3
+/* Scan period of the park timer while anything is parked. */
+#define BTRFS_RBIO_PARK_SCAN_MS 3
+
#define BTRFS_STRIPE_HASH_TABLE_BITS 11
static void dump_bioc(const struct btrfs_fs_info *fs_info, const struct btrfs_io_context *bioc)
struct list_head stripe_cache;
spinlock_t cache_lock;
int cache_size;
+ /*
+ * Parked partial write rbios (RBIO_PARKED_BIT), in park order.
+ * parked_lock nests inside the hash and bio_list locks.
+ */
+ spinlock_t parked_lock;
+ struct list_head parked;
+ struct delayed_work parked_work;
+ struct btrfs_fs_info *fs_info;
struct btrfs_stripe_hash table[];
};
static void rmw_rbio_work(struct work_struct *work);
static void rmw_rbio_work_locked(struct work_struct *work);
+static void parked_rbios_timeout_work(struct work_struct *work);
static void index_rbio_pages(struct btrfs_raid_bio *rbio);
static int alloc_rbio_pages(struct btrfs_raid_bio *rbio);
spin_lock_init(&table->cache_lock);
INIT_LIST_HEAD(&table->stripe_cache);
+ spin_lock_init(&table->parked_lock);
+ INIT_LIST_HEAD(&table->parked);
+ INIT_DELAYED_WORK(&table->parked_work, parked_rbios_timeout_work);
+ table->fs_info = info;
h = table->table;
{
if (!info->stripe_hash_table)
return;
+ cancel_delayed_work_sync(&info->stripe_hash_table->parked_work);
+ WARN_ON(!list_empty(&info->stripe_hash_table->parked));
btrfs_clear_rbio_cache(info);
kvfree(info->stripe_hash_table);
info->stripe_hash_table = NULL;
return 1;
}
+/*
+ * Full-stripe write batching for stripe-exclusive allocation.
+ *
+ * A partial write rbio whose stripe belongs to an open stripe run is parked:
+ * it keeps the stripe lock but its RMW is not started, so the writes that
+ * fill the rest of the stripe -- guaranteed to arrive before the run closes,
+ * or not at all -- merge into it through lock_stripe_add() instead of each
+ * paying a read-modify-write. The moment the data bitmap fills, the rbio is
+ * submitted as a single full-stripe write: one parity computation, no reads.
+ *
+ * Completion accounting can never be the flush trigger -- parked bios have
+ * not been submitted, and preallocated or discarded allocations produce no
+ * bios at all -- so everything that does not fill is flushed by the run
+ * retirement paths (before the commit's IO drain, which waits on the very
+ * bios parked here), by ordered-extent waiters, and by a timer that bounds
+ * both latency and the memory the parked pages pin.
+ */
+
+/* Caller holds table->parked_lock. */
+static void rbio_unpark_locked(struct btrfs_raid_bio *rbio)
+{
+ list_del_init(&rbio->parked_node);
+ clear_bit(RBIO_PARKED_BIT, &rbio->flags);
+}
+
+/* Does any bio gathered in @rbio carry a sync hint (a blocked waiter)? */
+static bool rbio_has_sync_bio(struct btrfs_raid_bio *rbio)
+{
+ struct bio *bio;
+
+ bio_list_for_each(bio, &rbio->bio_list)
+ if (bio->bi_opf & REQ_SYNC)
+ return true;
+ return false;
+}
+
+/*
+ * Park a partial write rbio that owns its stripe lock, if its stripe is
+ * covered by an open stripe run. Returns true if the rbio was parked (or
+ * concurrently filled and requeued); false if the caller should proceed
+ * with the RMW itself.
+ */
+static bool rbio_try_park(struct btrfs_raid_bio *rbio)
+{
+ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+ struct btrfs_stripe_hash_table *table = fs_info->stripe_hash_table;
+ unsigned int timeout_ms = BTRFS_RBIO_PARK_TIMEOUT_MS;
+
+ if (!btrfs_stripe_in_open_run(fs_info, rbio->bioc->full_stripe_logical))
+ return false;
+
+ spin_lock(&rbio->bio_list_lock);
+ if (rbio_has_sync_bio(rbio))
+ timeout_ms = BTRFS_RBIO_PARK_SYNC_TIMEOUT_MS;
+ spin_unlock(&rbio->bio_list_lock);
+
+ spin_lock(&table->parked_lock);
+ set_bit(RBIO_PARKED_BIT, &rbio->flags);
+ rbio->park_deadline = jiffies + msecs_to_jiffies(timeout_ms);
+ list_add_tail(&rbio->parked_node, &table->parked);
+ spin_unlock(&table->parked_lock);
+ queue_delayed_work(system_percpu_wq, &table->parked_work,
+ msecs_to_jiffies(BTRFS_RBIO_PARK_SCAN_MS));
+
+ /*
+ * A merge may have filled us between the caller's fullness check and
+ * the park above; reclaim the park so the full stripe goes straight
+ * down. If the merger's unpark won instead, it queued the work.
+ */
+ if (rbio_is_full(rbio)) {
+ spin_lock(&table->parked_lock);
+ if (test_bit(RBIO_PARKED_BIT, &rbio->flags)) {
+ rbio_unpark_locked(rbio);
+ spin_unlock(&table->parked_lock);
+ return false;
+ }
+ spin_unlock(&table->parked_lock);
+ }
+ return true;
+}
+
+/*
+ * A merge just filled a parked rbio: unpark it and start its (now read-free)
+ * write. Callers may hold the stripe hash and bio_list locks; parked_lock
+ * nests inside both.
+ */
+static void unpark_full_rbio(struct btrfs_raid_bio *rbio)
+{
+ struct btrfs_stripe_hash_table *table =
+ rbio->bioc->fs_info->stripe_hash_table;
+
+ spin_lock(&table->parked_lock);
+ if (!test_bit(RBIO_PARKED_BIT, &rbio->flags)) {
+ spin_unlock(&table->parked_lock);
+ return;
+ }
+ rbio_unpark_locked(rbio);
+ spin_unlock(&table->parked_lock);
+ start_async_work(rbio, rmw_rbio_work_locked);
+}
+
+/*
+ * Flush every parked rbio overlapping [start, start + num_bytes) down its
+ * normal RMW path. Called by run retirement (before the commit's IO drain),
+ * ordered-extent waiters, and the park timer.
+ */
+void btrfs_flush_parked_rbios(struct btrfs_fs_info *fs_info, u64 start,
+ u64 num_bytes)
+{
+ struct btrfs_stripe_hash_table *table = fs_info->stripe_hash_table;
+ struct btrfs_raid_bio *rbio;
+ struct btrfs_raid_bio *tmp;
+ LIST_HEAD(flush);
+
+ if (!table || list_empty_careful(&table->parked))
+ return;
+ spin_lock(&table->parked_lock);
+ list_for_each_entry_safe(rbio, tmp, &table->parked, parked_node) {
+ u64 rbio_start = rbio->bioc->full_stripe_logical;
+ u64 rbio_len = (u64)rbio->nr_data * BTRFS_STRIPE_LEN;
+
+ if (rbio_start + rbio_len <= start ||
+ (num_bytes != U64_MAX && rbio_start >= start + num_bytes))
+ continue;
+ rbio_unpark_locked(rbio);
+ list_add_tail(&rbio->parked_node, &flush);
+ }
+ spin_unlock(&table->parked_lock);
+
+ list_for_each_entry_safe(rbio, tmp, &flush, parked_node) {
+ list_del_init(&rbio->parked_node);
+ start_async_work(rbio, rmw_rbio_work_locked);
+ }
+}
+
+/* Flush parked rbios whose deadline has passed; re-arm while any remain. */
+static void parked_rbios_timeout_work(struct work_struct *work)
+{
+ struct btrfs_stripe_hash_table *table =
+ container_of(work, struct btrfs_stripe_hash_table,
+ parked_work.work);
+ struct btrfs_raid_bio *rbio;
+ struct btrfs_raid_bio *tmp;
+ LIST_HEAD(flush);
+ bool rearm = false;
+
+ spin_lock(&table->parked_lock);
+ list_for_each_entry_safe(rbio, tmp, &table->parked, parked_node) {
+ /* Deadlines are not ordered (sync parks are shorter). */
+ if (time_before(jiffies, rbio->park_deadline)) {
+ rearm = true;
+ continue;
+ }
+ rbio_unpark_locked(rbio);
+ list_add_tail(&rbio->parked_node, &flush);
+ }
+ spin_unlock(&table->parked_lock);
+
+ list_for_each_entry_safe(rbio, tmp, &flush, parked_node) {
+ list_del_init(&rbio->parked_node);
+ start_async_work(rbio, rmw_rbio_work_locked);
+ }
+ if (rearm)
+ queue_delayed_work(system_percpu_wq, &table->parked_work,
+ msecs_to_jiffies(BTRFS_RBIO_PARK_SCAN_MS));
+}
+
+/* Return the sector index for @stripe_nr and @sector_nr. */
static unsigned int rbio_stripe_sector_index(const struct btrfs_raid_bio *rbio,
unsigned int stripe_nr,
unsigned int sector_nr)
/* Can we merge into the lock owner? */
if (rbio_can_merge(cur, rbio)) {
+ bool sync = rbio_has_sync_bio(rbio);
+
merge_rbio(cur, rbio);
+ /*
+ * Our bios may have completed a parked stripe. We
+ * hold cur->bio_list_lock, so test fullness directly
+ * (rbio_is_full() would self-deadlock on it). A sync
+ * bio merging into a still-partial parked rbio pulls
+ * its flush deadline in: a waiter is now blocked on
+ * this stripe.
+ */
+ if (test_bit(RBIO_PARKED_BIT, &cur->flags)) {
+ if (cur->bio_list_bytes ==
+ cur->nr_data * BTRFS_STRIPE_LEN) {
+ unpark_full_rbio(cur);
+ } else if (sync) {
+ struct btrfs_stripe_hash_table *table =
+ cur->bioc->fs_info->stripe_hash_table;
+ unsigned long dl = jiffies +
+ msecs_to_jiffies(BTRFS_RBIO_PARK_SYNC_TIMEOUT_MS);
+
+ spin_lock(&table->parked_lock);
+ if (test_bit(RBIO_PARKED_BIT,
+ &cur->flags) &&
+ time_before(dl, cur->park_deadline))
+ cur->park_deadline = dl;
+ spin_unlock(&table->parked_lock);
+ }
+ }
spin_unlock(&cur->bio_list_lock);
freeit = rbio;
ret = 1;
spin_lock_init(&rbio->bio_list_lock);
INIT_LIST_HEAD(&rbio->stripe_cache);
INIT_LIST_HEAD(&rbio->hash_list);
+ INIT_LIST_HEAD(&rbio->parked_node);
btrfs_get_bioc(bioc);
rbio->bioc = bioc;
rbio->nr_pages = num_pages;
struct btrfs_raid_bio *rbio;
rbio = container_of(work, struct btrfs_raid_bio, work);
- if (lock_stripe_add(rbio) == 0)
+ if (lock_stripe_add(rbio) == 0) {
+ if (!rbio_is_full(rbio) && rbio_try_park(rbio))
+ return;
rmw_rbio(rbio);
+ }
}
static void rmw_rbio_work_locked(struct work_struct *work)
projects / linux / commitdiff