*/
#define RBIO_PINNED_BIT 9
-/* At most this many pinned stripes; normally about one per stripe run. */
+/*
+ * At most this many pinned stripes: about as many as stripes with writes in
+ * flight, but fsync-heavy work caches a partial stripe on nearly every write.
+ * A pin goes stale when its run finishes the stripe and is only uncounted
+ * when a sweep, an eviction or a re-cache finds it, so pinning past
+ * RBIO_PIN_LOW sweeps first, and if the pins left are still live, asks for a
+ * commit: the commit retires the runs (btrfs_retire_open_stripes(), and
+ * btrfs_retire_meta_stripes() for metadata), which makes their pins
+ * unnecessary, and its sweep releases them.
+ */
#define RBIO_PIN_MAX 256
+#define RBIO_PIN_LOW (RBIO_PIN_MAX * 3 / 4)
+/* At most one sweep per this interval from the pin path (commits always sweep). */
+#define RBIO_PIN_SWEEP_INTERVAL (HZ / 20)
#define RBIO_CACHE_SIZE 1024
int cache_size;
/* Cached rbios with RBIO_PINNED_BIT set, live or not. */
int nr_pinned;
+ /* Last pin sweep, and the generation the last commit request was for. */
+ unsigned long pin_sweep_jiffies;
+ u64 pin_commit_gen;
/*
* Parked partial write rbios (RBIO_PARKED_BIT), in park order.
* parked_lock nests inside the hash and bio_list locks.
}
}
+static void rbio_stat_hwm(atomic64_t *hwm, int now)
+{
+ /* Only ever raised under the cache lock: no lost update. */
+ if (now > atomic64_read(hwm))
+ atomic64_set(hwm, now);
+}
+
/*
* Mark @rbio pinned or not; caller holds the cache lock and @rbio is in the
- * cache.
+ * cache. A pin refused at RBIO_PIN_MAX is counted at every attempt.
*/
static void rbio_set_pin(struct btrfs_stripe_hash_table *table,
struct btrfs_raid_bio *rbio, bool pin)
{
if (pin && table->nr_pinned >= RBIO_PIN_MAX &&
- !test_bit(RBIO_PINNED_BIT, &rbio->flags))
+ !test_bit(RBIO_PINNED_BIT, &rbio->flags)) {
+ atomic64_inc(&table->fs_info->stripe_park_stats.rbio_pin_refused);
pin = false;
+ }
if (pin) {
- if (!test_and_set_bit(RBIO_PINNED_BIT, &rbio->flags))
+ if (!test_and_set_bit(RBIO_PINNED_BIT, &rbio->flags)) {
table->nr_pinned += 1;
+ rbio_stat_hwm(&table->fs_info->stripe_park_stats.rbio_pinned_hwm,
+ table->nr_pinned);
+ }
atomic64_inc(&rbio->bioc->fs_info->stripe_park_stats.rbio_pinned);
} else if (test_and_clear_bit(RBIO_PINNED_BIT, &rbio->flags)) {
table->nr_pinned -= 1;
/* __remove_rbio_from_cache() drops a stale pin's count. */
__remove_rbio_from_cache(found);
}
+ /* Above the limit only by live pins, which the prune keeps. */
+ rbio_stat_hwm(&table->fs_info->stripe_park_stats.rbio_cache_hwm,
+ table->cache_size);
+}
+
+/*
+ * Clear the pins no longer needed (their run has written all of the stripe,
+ * or is gone). Caller holds the cache lock, the same context the prune
+ * re-checks pins in. Returns how many pins are left: all of them live.
+ */
+static int rbio_unpin_stale(struct btrfs_stripe_hash_table *table)
+{
+ struct btrfs_raid_bio *rbio;
+ int swept = 0;
+
+ table->pin_sweep_jiffies = jiffies;
+ list_for_each_entry(rbio, &table->stripe_cache, stripe_cache) {
+ if (!test_bit(RBIO_PINNED_BIT, &rbio->flags) || rbio_needs_pin(rbio))
+ continue;
+ clear_bit(RBIO_PINNED_BIT, &rbio->flags);
+ table->nr_pinned -= 1;
+ swept++;
+ }
+ if (swept)
+ atomic64_add(swept, &table->fs_info->stripe_park_stats.rbio_pin_swept);
+ return table->nr_pinned;
+}
+
+/*
+ * About to pin with the pins running low: sweep the stale ones, and if the
+ * live ones are still many, ask for a commit (once per transaction), which
+ * ends every run and so every pin. Caller holds the cache lock.
+ */
+static void rbio_pins_low(struct btrfs_stripe_hash_table *table)
+{
+ struct btrfs_fs_info *fs_info = table->fs_info;
+ u64 gen;
+
+ /*
+ * Rate limited: a sweep walks every pin, and only a commit frees live
+ * ones.
+ */
+ if (!time_after(jiffies, table->pin_sweep_jiffies + RBIO_PIN_SWEEP_INTERVAL))
+ return;
+ if (rbio_unpin_stale(table) < RBIO_PIN_LOW)
+ return;
+ gen = btrfs_get_fs_generation(fs_info);
+ if (table->pin_commit_gen == gen)
+ return;
+ table->pin_commit_gen = gen;
+ atomic64_inc(&fs_info->stripe_park_stats.rbio_pin_commit_requests);
+ /* As btrfs_commit_transaction_async(): the kthread commits now. */
+ set_bit(BTRFS_FS_COMMIT_TRANS, &fs_info->flags);
+ wake_up_process(fs_info->transaction_kthread);
+}
+
+/*
+ * A commit has retired every open stripe run: release the pins that leaves
+ * stale. Pins of runs opened since then are re-checked, not dropped.
+ */
+void btrfs_rbio_unpin_stale(struct btrfs_fs_info *fs_info)
+{
+ struct btrfs_stripe_hash_table *table = fs_info->stripe_hash_table;
+
+ if (!table || !btrfs_test_opt(fs_info, STRIPE_ALLOC))
+ return;
+ spin_lock(&table->cache_lock);
+ if (table->nr_pinned)
+ rbio_unpin_stale(table);
+ spin_unlock(&table->cache_lock);
+}
+
+/*
+ * Cached and pinned rbios now, for sysfs stripe_park_stats. @live re-checks
+ * each pin against its run, as the prune does: a pin whose run has finished
+ * writing the stripe is still counted in nr_pinned until a sweep, an
+ * eviction or a re-cache finds it.
+ */
+void btrfs_rbio_cache_usage(struct btrfs_fs_info *fs_info, int *cached,
+ int *pinned, int *live)
+{
+ struct btrfs_stripe_hash_table *table = fs_info->stripe_hash_table;
+ struct btrfs_raid_bio *rbio;
+
+ *cached = 0;
+ *pinned = 0;
+ *live = 0;
+ if (!table)
+ return;
+ spin_lock(&table->cache_lock);
+ *cached = table->cache_size;
+ *pinned = table->nr_pinned;
+ list_for_each_entry(rbio, &table->stripe_cache, stripe_cache)
+ if (rbio_pin_live(rbio))
+ *live += 1;
+ spin_unlock(&table->cache_lock);
}
/*
table = rbio->bioc->fs_info->stripe_hash_table;
spin_lock(&table->cache_lock);
+ if (pin && table->nr_pinned >= RBIO_PIN_LOW &&
+ !test_bit(RBIO_PINNED_BIT, &rbio->flags))
+ rbio_pins_low(table);
spin_lock(&rbio->bio_list_lock);
/* bump our ref if we were not in the list before */
* count that should be zero.
*/
atomic64_inc(&fs_info->stripe_park_stats.data_rmw);
+ if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA))
+ atomic64_inc(&fs_info->stripe_park_stats.meta_rmw_reads);
/* A parked-then-flushed write that still had to read. */
if (test_bit(RBIO_WAS_PARKED_BIT, &rbio->flags))
atomic64_inc(&fs_info->stripe_park_stats.rmw_reads);
#include "misc.h"
#include "fs.h"
#include "accessors.h"
+#include "raid56.h"
/*
* Structure name Path
struct kobj_attribute *a, char *buf)
{
struct btrfs_fs_info *fs_info = to_fs_info(kobj);
+ int cached, pinned, live;
+ btrfs_rbio_cache_usage(fs_info, &cached, &pinned, &live);
return sysfs_emit(buf,
"parked %lld\n"
"merged_bytes %lld\n"
"stuck %lld\n"
"padded %lld\n"
"padded_unwritten %lld\n"
- "rmw_reads %lld\n",
+ "rmw_reads %lld\n"
+ "rbio_pin_refused %lld\n"
+ "rbio_pinned_now %d\n"
+ "rbio_pinned_live %d\n"
+ "rbio_pinned_hwm %lld\n"
+ "rbio_cache_now %d\n"
+ "rbio_cache_hwm %lld\n"
+ "rbio_pin_swept %lld\n"
+ "rbio_pin_commit_requests %lld\n"
+ "meta_rmw_reads %lld\n",
atomic64_read(&fs_info->stripe_park_stats.parked),
atomic64_read(&fs_info->stripe_park_stats.merged_bytes),
atomic64_read(&fs_info->stripe_park_stats.filled),
atomic64_read(&fs_info->stripe_park_stats.stuck),
atomic64_read(&fs_info->stripe_park_stats.padded),
atomic64_read(&fs_info->stripe_park_stats.padded_unwritten),
- atomic64_read(&fs_info->stripe_park_stats.rmw_reads));
+ atomic64_read(&fs_info->stripe_park_stats.rmw_reads),
+ atomic64_read(&fs_info->stripe_park_stats.rbio_pin_refused),
+ pinned,
+ live,
+ atomic64_read(&fs_info->stripe_park_stats.rbio_pinned_hwm),
+ cached,
+ atomic64_read(&fs_info->stripe_park_stats.rbio_cache_hwm),
+ atomic64_read(&fs_info->stripe_park_stats.rbio_pin_swept),
+ atomic64_read(&fs_info->stripe_park_stats.rbio_pin_commit_requests),
+ atomic64_read(&fs_info->stripe_park_stats.meta_rmw_reads));
}
BTRFS_ATTR(, stripe_park_stats, btrfs_stripe_park_stats_show);