* were written), which are as safe to zero-fill as space above it.
*/
unsigned long *live;
+ /*
+ * Sectors of @live whose data has been written (raid56 write
+ * completions, btrfs_stripe_run_mark_written()); same size as @live.
+ * A stripe with live sectors not yet written is still waiting for
+ * writes, and its cached rbio stays pinned (raid56.c, cache_rbio()).
+ */
+ unsigned long *written;
u32 live_nbits; /* sectors @live covers; caps growth */
};
ASSERT(open_stripe_run_drained(run));
list_del(&run->list);
bitmap_free(run->live);
+ bitmap_free(run->written);
kfree(run);
/*
* Wholly free stripes inside this run's range were counted as trapped
/* Sleeps, so it has to happen before the locks below. */
new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
+ new_run->written = new_run->live_nbits ?
+ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
/*
* Install under fs_info->open_stripe_lock so that the block group
ret = 0;
out:
/* Unused on every path that did not install it, including errors. */
- if (new_run)
+ if (new_run) {
bitmap_free(new_run->live);
+ bitmap_free(new_run->written);
+ }
kfree(new_run);
return ret;
}
/* Sleeps, so it has to happen before the locks below. */
new_run->live = alloc_run_live_map(bg, len, &new_run->live_nbits);
+ new_run->written = new_run->live_nbits ?
+ bitmap_zalloc(new_run->live_nbits, GFP_NOFS) : NULL;
/* See the installation comment in btrfs_alloc_from_open_stripe(). */
spin_lock(&fs_info->open_stripe_lock);
ret = 0;
out:
/* Unused on every path that did not install it, including errors. */
- if (new_run)
+ if (new_run) {
bitmap_free(new_run->live);
+ bitmap_free(new_run->written);
+ }
kfree(new_run);
return ret;
}
* the commit drain already waits for and everything above it is dead space
* padding may fill.
*/
+/*
+ * The part of [*start, *start + *len) that @run owns and tracks in its
+ * written map: inside [run->start, run->end) -- the map itself reaches
+ * further (it spans at least BTRFS_STRIPE_RUN_LIVE_STRIPES stripes from
+ * run->start, past run->end into later runs' ranges), so it must never be
+ * consulted outside the run's own range -- and inside the map. Nodatacow
+ * runs keep no written map: they are rewritten in place for as long as the
+ * inode lives and are never complete. Caller holds stripe_run_lock.
+ */
+static bool run_written_range(const struct btrfs_open_stripe_run *run,
+ u64 *start, u64 *len)
+{
+ const u32 bits = run->bg->fs_info->sectorsize_bits;
+ u64 lo, hi;
+
+ if (!run->live || !run->written || !run->live_nbits)
+ return false;
+ lo = max(*start, run->start);
+ hi = min3(*start + *len, run->end,
+ run->start + ((u64)run->live_nbits << bits));
+ if (lo >= hi)
+ return false;
+ *start = lo;
+ *len = hi - lo;
+ return true;
+}
+
+/*
+ * A raid56 write of [start, start + len) has completed: mark it written in
+ * the run that owns it. Sectors that are not live in that run (freed while
+ * the write was in flight, or a bookkeeping bug) are counted in
+ * written_not_live: the pad oracle trusts this map to say which allocated
+ * sectors hold data, so a write it misses could be zeroed.
+ */
+void btrfs_stripe_run_mark_written(struct btrfs_fs_info *fs_info, u64 start,
+ u64 len)
+{
+ const u32 bits = fs_info->sectorsize_bits;
+ struct btrfs_block_group *bg;
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ u32 not_live = 0;
+
+ if (list_empty_careful(&fs_info->open_stripe_bgs))
+ return;
+ bg = btrfs_lookup_block_group(fs_info, start);
+ if (!bg)
+ return;
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ u64 s = start, l = len;
+ u32 first, nr, i;
+
+ if (!run_written_range(run, &s, &l))
+ continue;
+ first = (s - run->start) >> bits;
+ nr = l >> bits;
+ for (i = first; i < first + nr; i++)
+ if (!test_bit(i, run->live))
+ not_live++;
+ bitmap_set(run->written, first, nr);
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ btrfs_put_block_group(bg);
+ if (not_live)
+ atomic64_add(not_live, &fs_info->stripe_park_stats.written_not_live);
+}
+
+/* Has @run allocated sectors in [start, start + len) it has not yet written? */
+static bool run_unwritten_in(struct btrfs_open_stripe_run *run, u64 start,
+ u64 len)
+{
+ const u32 bits = run->bg->fs_info->sectorsize_bits;
+ u32 first, last, i;
+
+ if (!run_written_range(run, &start, &len))
+ return false;
+ first = (start - run->start) >> bits;
+ last = first + (len >> bits);
+ for (i = first; i < last; i++)
+ if (test_bit(i, run->live) && !test_bit(i, run->written))
+ return true;
+ return false;
+}
+
+bool btrfs_stripe_run_incomplete(struct btrfs_fs_info *fs_info,
+ u64 stripe_start, u64 stripe_len)
+{
+ struct btrfs_block_group *bg;
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ bool ret = false;
+
+ if (list_empty_careful(&fs_info->open_stripe_bgs))
+ return false;
+ bg = btrfs_lookup_block_group(fs_info, stripe_start);
+ if (!bg)
+ return false;
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (run->open && run->offset > stripe_start &&
+ run->offset < stripe_start + stripe_len) {
+ ret = true;
+ break;
+ }
+ if (run_unwritten_in(run, stripe_start, stripe_len)) {
+ ret = true;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ btrfs_put_block_group(bg);
+ return ret;
+}
+
bool btrfs_stripe_in_open_run(struct btrfs_fs_info *fs_info, u64 logical)
{
struct btrfs_block_group *bg;
/*
* Pointer-based variant of btrfs_open_stripe_write_done() for callers that
- * recorded the run at allocation or ordered extent creation time; safe from
- * bio end_io context. The run pointer stays valid until the reporter's own
- * bytes are reported: a run with inflight bytes is never freed.
+ * recorded the run at allocation or ordered extent creation time. The run
+ * pointer stays valid until the reporter's own bytes are reported: a run
+ * with inflight bytes is never freed.
+ *
+ * An ordered extent's data IO is over, whether or not all of it reached the
+ * disk: stop counting it as in flight, and mark its sectors written. A
+ * sector whose write failed or was never issued (truncation, a short direct
+ * write, error teardown) will not be written now. Marking it keeps a log
+ * settle from waiting for it forever, and a written sector is never padded,
+ * so a later partial write to the stripe reads it instead of zeroing it.
*/
void btrfs_open_stripe_write_done_run(struct btrfs_open_stripe_run *run,
- u64 num_bytes)
+ u64 start, u64 num_bytes)
{
struct btrfs_block_group *bg = run->bg;
+ const u32 bits = bg->fs_info->sectorsize_bits;
unsigned long flags;
+ u64 s = start;
+ u64 l = num_bytes;
spin_lock_irqsave(&bg->stripe_run_lock, flags);
ASSERT(run->inflight_bytes >= num_bytes);
run->inflight_bytes -= num_bytes;
+ if (run_written_range(run, &s, &l))
+ bitmap_set(run->written, (s - run->start) >> bits, l >> bits);
+ /* Under the lock: the run and the group may be gone after it. */
+ wake_up_var(&bg->open_stripe_runs);
if (open_stripe_run_drained(run) && !run->retained)
free_open_stripe_run(bg, run);
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
*/
#define RBIO_SYNC_PARK_BIT 8
+/*
+ * Set while a cached rbio holds a stripe that its run has not finished
+ * writing: the cache prune skips it, so the run's next write into the stripe
+ * takes the part already written from memory instead of reading it back
+ * from the devices (see rbio_needs_pin()).
+ */
+#define RBIO_PINNED_BIT 9
+
+/* At most this many pinned stripes; normally about one per stripe run. */
+#define RBIO_PIN_MAX 256
+
#define RBIO_CACHE_SIZE 1024
/* Scan period of the park timer while anything is parked. */
struct list_head stripe_cache;
spinlock_t cache_lock;
int cache_size;
+ /* Cached rbios with RBIO_PINNED_BIT set, live or not. */
+ int nr_pinned;
/*
* Parked partial write rbios (RBIO_PARKED_BIT), in park order.
* parked_lock nests inside the hash and bio_list locks.
if (test_and_clear_bit(RBIO_CACHE_BIT, &rbio->flags)) {
list_del_init(&rbio->stripe_cache);
table->cache_size -= 1;
+ if (test_and_clear_bit(RBIO_PINNED_BIT, &rbio->flags))
+ table->nr_pinned -= 1;
freeit = true;
/* if the bio list isn't empty, this rbio is
info->stripe_hash_table = NULL;
}
+/*
+ * A stripe its open run has not finished writing is written more than once:
+ * each write that ends inside the stripe leaves the rest to a later write.
+ * That later write cannot pad (the earlier sectors are live) and would read
+ * them back, folding into parity whatever the devices return before the
+ * csum tree covers those sectors. Keep such a stripe cached instead, so the
+ * later write steals the pages it needs (lock_stripe_add()) and reads
+ * nothing.
+ *
+ * That is the stripe holding the run's allocation frontier, and any stripe
+ * behind it whose allocated sectors are still being written
+ * (btrfs_stripe_run_incomplete()). Each prune re-checks the pin against
+ * the run, and the run's next write into the stripe replaces the entry.
+ */
+static bool rbio_needs_pin(const struct btrfs_raid_bio *rbio)
+{
+ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+
+ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+ return false;
+ return btrfs_stripe_run_incomplete(fs_info,
+ rbio->bioc->full_stripe_logical,
+ (u64)rbio->nr_data * BTRFS_STRIPE_LEN);
+}
+
+/* Re-checked at every prune: the pin ends when its run has written it all. */
+static bool rbio_pin_live(const struct btrfs_raid_bio *rbio)
+{
+ return test_bit(RBIO_PINNED_BIT, &rbio->flags) && rbio_needs_pin(rbio);
+}
+
+/*
+ * A write completed: tell the stripe runs which sectors now hold their data,
+ * as contiguous ranges of the data sectors this rbio's bios covered.
+ */
+static void rbio_mark_written(struct btrfs_raid_bio *rbio)
+{
+ struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
+ const u32 sectorsize = fs_info->sectorsize;
+ const int nr = rbio->nr_data * rbio->stripe_nsectors;
+ int i, first = -1;
+
+ if (!btrfs_test_opt(fs_info, STRIPE_ALLOC))
+ return;
+ for (i = 0; i <= nr; i++) {
+ const bool covered = i < nr && rbio->bio_paddrs[i * rbio->sector_nsteps] != INVALID_PADDR;
+
+ if (covered && first < 0)
+ first = i;
+ if (!covered && first >= 0) {
+ btrfs_stripe_run_mark_written(fs_info,
+ rbio->bioc->full_stripe_logical +
+ (u64)first * sectorsize,
+ (u64)(i - first) * sectorsize);
+ first = -1;
+ }
+ }
+}
+
+/*
+ * Mark @rbio pinned or not; caller holds the cache lock and @rbio is in the
+ * cache.
+ */
+static void rbio_set_pin(struct btrfs_stripe_hash_table *table,
+ struct btrfs_raid_bio *rbio, bool pin)
+{
+ if (pin && table->nr_pinned >= RBIO_PIN_MAX &&
+ !test_bit(RBIO_PINNED_BIT, &rbio->flags))
+ pin = false;
+ if (pin) {
+ if (!test_and_set_bit(RBIO_PINNED_BIT, &rbio->flags))
+ table->nr_pinned += 1;
+ atomic64_inc(&rbio->bioc->fs_info->stripe_park_stats.rbio_pinned);
+ } else if (test_and_clear_bit(RBIO_PINNED_BIT, &rbio->flags)) {
+ table->nr_pinned -= 1;
+ }
+}
+
+/*
+ * Evict from the cold end until the cache is within @limit, keeping @rbio
+ * (just cached) and live pins. Caller holds the cache lock.
+ */
+static void rbio_cache_prune(struct btrfs_stripe_hash_table *table,
+ struct btrfs_raid_bio *rbio, unsigned int limit)
+{
+ struct btrfs_raid_bio *found, *tmp;
+
+ list_for_each_entry_safe_reverse(found, tmp, &table->stripe_cache,
+ stripe_cache) {
+ if (table->cache_size <= limit)
+ break;
+ if (found == rbio || rbio_pin_live(found))
+ continue;
+ /* __remove_rbio_from_cache() drops a stale pin's count. */
+ __remove_rbio_from_cache(found);
+ }
+}
+
/*
* insert an rbio into the stripe cache. It
* must have already been prepared by calling
static void cache_rbio(struct btrfs_raid_bio *rbio)
{
struct btrfs_stripe_hash_table *table;
+ bool pin;
if (!test_bit(RBIO_CACHE_READY_BIT, &rbio->flags))
return;
+ pin = rbio_needs_pin(rbio);
table = rbio->bioc->fs_info->stripe_hash_table;
spin_lock(&table->cache_lock);
list_add(&rbio->stripe_cache, &table->stripe_cache);
table->cache_size += 1;
}
+ rbio_set_pin(table, rbio, pin);
spin_unlock(&rbio->bio_list_lock);
- if (table->cache_size > RBIO_CACHE_SIZE) {
- struct btrfs_raid_bio *found;
-
- found = list_last_entry(&table->stripe_cache,
- struct btrfs_raid_bio,
- stripe_cache);
-
- if (found != rbio)
- __remove_rbio_from_cache(found);
- }
+ rbio_cache_prune(table, rbio, RBIO_CACHE_SIZE);
spin_unlock(&table->cache_lock);
}
rbio->csum_buf = NULL;
rbio->csum_bitmap = NULL;
+ /* Before unlock_stripe()'s cache decision: see rbio_needs_pin(). */
+ if (rbio->operation == BTRFS_RBIO_WRITE && status == BLK_STS_OK)
+ rbio_mark_written(rbio);
+
/*
* Clear the data bitmap, as the rbio may be cached for later usage.
* do this before before unlock_stripe() so there will be no new bio