return settled;
}
+/*
+ * Remember, on the logging inode, a logged extent that lives in the final
+ * partial stripe of one of the inode's recently closed runs, so that the
+ * next fsync can re-dirty its file range and carry the data forward into
+ * the then-current run -- letting the old partial stripe free whole. A
+ * zero @file_len means the caller does not want this extent carried
+ * (e.g. compressed extents, which would recompress). @ps_len is nonzero
+ * when the settling walk could see the run and computed its final partial
+ * stripe. Every table here is small and bounded; on overflow the extent
+ * simply is not carried and its stripe frees the slow way. Caller holds
+ * the inode's log_mutex, which serializes against the fsync-entry consumer.
+ */
+static void log_carry_record(struct btrfs_inode *inode, u64 file_start,
+ u64 file_len, u64 bytenr, u64 num_bytes,
+ u64 ps_start, u64 ps_len)
+{
+ struct btrfs_log_carry *carry = inode->log_carry;
+ int n, i;
+
+ if (!carry) {
+ carry = kzalloc(sizeof(*carry), GFP_NOFS);
+ if (!carry)
+ return;
+ inode->log_carry = carry;
+ }
+ if (ps_len) {
+ n = min_t(int, carry->nr_ps, BTRFS_LOG_CARRY_NR_PS);
+ for (i = 0; i < n; i++)
+ if (carry->ps_start[i] == ps_start)
+ break;
+ if (i == n) {
+ int slot = carry->nr_ps % BTRFS_LOG_CARRY_NR_PS;
+
+ carry->ps_start[slot] = ps_start;
+ carry->ps_len[slot] = ps_len;
+ carry->nr_ps++;
+ }
+ }
+ if (!file_len || carry->nr_ranges >= BTRFS_LOG_CARRY_NR_RANGES)
+ return;
+ n = min_t(int, carry->nr_ps, BTRFS_LOG_CARRY_NR_PS);
+ for (i = 0; i < n; i++)
+ if (bytenr < carry->ps_start[i] + carry->ps_len[i] &&
+ bytenr + num_bytes > carry->ps_start[i])
+ break;
+ if (i == n)
+ return;
+ for (i = 0; i < carry->nr_ranges; i++)
+ if (carry->range[i].start == file_start &&
+ carry->range[i].len == file_len)
+ return;
+ carry->range[carry->nr_ranges].start = file_start;
+ carry->range[carry->nr_ranges].len = file_len;
+ carry->range[carry->nr_ranges].disk_bytenr = bytenr;
+ carry->nr_ranges++;
+}
+
/*
* A log commit is about to make the extent at [bytenr, bytenr + num_bytes)
* durable. Once the log super lands, no later write may extend or RMW the
* the wait is bio flight time plus the parked-write deadline that the flush
* below short-circuits. It is a pure data-IO wait (no transaction joins),
* safe under the inode log mutex and a running transaction handle.
+ *
+ * [file_start, file_start + file_len) is the extent's file range, recorded
+ * for carry-forward when the extent sits in the run's final partial stripe;
+ * file_len == 0 opts the extent out of carrying (settling still happens).
*/
-void btrfs_log_settle_stripes(struct btrfs_fs_info *fs_info, u64 bytenr,
- u64 num_bytes)
+void btrfs_log_settle_stripes(struct btrfs_inode *inode, u64 file_start,
+ u64 file_len, u64 bytenr, u64 num_bytes)
{
+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct btrfs_block_group *bg;
struct btrfs_open_stripe_run *run;
unsigned long flags;
u64 tail_len = 0;
u64 flush_start = 0;
u64 flush_len = 0;
+ u64 ps_start = 0;
+ u64 ps_len = 0;
if (list_empty_careful(&fs_info->open_stripe_bgs))
return;
spin_lock_irqsave(&bg->stripe_run_lock, flags);
list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ u64 filled;
+
if (bytenr < run->start || bytenr >= run->offset)
continue;
flush_start = run->start;
flush_len = run->end - run->start;
+ /*
+ * The run's final partial stripe, whether this call closes
+ * the run or an earlier settle already did (offset froze at
+ * close). Runs start stripe-aligned, so alignment relative
+ * to run->start is stripe alignment.
+ */
+ filled = run->offset - run->start;
+ if (!IS_ALIGNED(filled, bg->full_stripe_len)) {
+ ps_start = run->start +
+ round_down(filled, bg->full_stripe_len);
+ ps_len = bg->full_stripe_len;
+ }
if (run->open) {
int class;
int band;
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ log_carry_record(inode, file_start, file_len, bytenr, num_bytes,
+ ps_start, ps_len);
+
if (tail_len)
btrfs_add_free_space(bg, tail_start, tail_len);
if (flush_len) {
bool btrfs_stripe_in_open_run(struct btrfs_fs_info *fs_info, u64 logical);
bool btrfs_stripe_run_range_usable(struct btrfs_block_group *bg,
u64 run_start, u64 *run_len);
-void btrfs_log_settle_stripes(struct btrfs_fs_info *fs_info, u64 bytenr,
- u64 num_bytes);
+void btrfs_log_settle_stripes(struct btrfs_inode *inode, u64 file_start,
+ u64 file_len, u64 bytenr, u64 num_bytes);
int btrfs_stripe_alloc_check_support(struct btrfs_fs_info *fs_info);
int btrfs_enable_stripe_alloc(struct btrfs_fs_info *fs_info);
void btrfs_disable_stripe_alloc(struct btrfs_fs_info *fs_info);
};
/* in memory btrfs inode */
+/*
+ * Carry-forward bookkeeping for a log-active inode under stripe-exclusive
+ * allocation. ps_start/ps_len remember the final partial stripes of the
+ * inode's recently closed private runs (a small ring; older entries are
+ * overwritten). range[] records the file ranges of logged extents living
+ * in those partial stripes, for the next fsync to re-dirty; disk_bytenr
+ * lets the consumer verify the range still maps there before carrying.
+ * Bounded loss: when the ring or the range table overflows, the extra
+ * extents simply stay put and their stripes free the slow way.
+ */
+#define BTRFS_LOG_CARRY_NR_PS 4
+#define BTRFS_LOG_CARRY_NR_RANGES 48
+struct btrfs_log_carry {
+ int nr_ps;
+ u64 ps_start[BTRFS_LOG_CARRY_NR_PS];
+ u64 ps_len[BTRFS_LOG_CARRY_NR_PS];
+ int nr_ranges;
+ struct {
+ u64 start;
+ u64 len;
+ u64 disk_bytenr;
+ } range[BTRFS_LOG_CARRY_NR_RANGES];
+};
+
struct btrfs_inode {
/* which subvolume this inode belongs to */
struct btrfs_root *root;
u64 ref_root_id;
};
+ /*
+ * Carry-forward state for a log-active inode (see
+ * BTRFS_INODE_LOG_ALLOC): the file ranges of this inode's extents
+ * that were left in the final partial stripe of a private stripe run
+ * closed at a log commit. The next fsync re-dirties them so they
+ * COW forward into the current run, letting the old partial stripe
+ * free whole. Lazily allocated; both recording (during logging) and
+ * consuming (at fsync entry) run under log_mutex.
+ */
+ struct btrfs_log_carry *log_carry;
+
/*
* Start of the last extent allocated from this inode's private
* LOG-class stripe run (see BTRFS_INODE_LOG_ALLOC). Advisory only:
return ret;
}
+/*
+ * Carry-forward for log-active inodes under stripe-exclusive allocation
+ * (see struct btrfs_log_carry). Re-dirty a file range recorded at the
+ * last log commit so the coming writeback COWs it into the inode's
+ * current private stripe run and the old partial stripe can free whole.
+ * The range is carried only while it still fully maps to the recorded
+ * disk bytenr; a range that was rewritten, truncated, hole-punched or
+ * already carried fails that check and is skipped -- skipping only costs
+ * the space optimization, never correctness. Modeled on
+ * defrag_one_range().
+ */
+static int carry_one_range(struct btrfs_inode *inode, u64 start, u64 len,
+ u64 disk_bytenr)
+{
+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
+ struct extent_changeset *data_reserved = NULL;
+ struct extent_state *cached_state = NULL;
+ struct extent_map *em;
+ struct folio **folios;
+ const unsigned int nr_pages = ((start + len - 1) >> PAGE_SHIFT) -
+ (start >> PAGE_SHIFT) + 1;
+ u64 cur;
+ int ret = 0;
+
+ ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
+ IS_ALIGNED(len, fs_info->sectorsize));
+
+ folios = kzalloc_objs(struct folio *, nr_pages, GFP_NOFS);
+ if (!folios)
+ return -ENOMEM;
+
+ cur = start;
+ for (int i = 0; cur < start + len && i < nr_pages; i++) {
+ folios[i] = defrag_prepare_one_folio(inode, cur >> PAGE_SHIFT);
+ if (IS_ERR(folios[i])) {
+ ret = PTR_ERR(folios[i]);
+ folios[i] = NULL;
+ goto free_folios;
+ }
+ cur = folio_pos(folios[i]) + folio_size(folios[i]);
+ }
+ for (int i = 0; i < nr_pages && folios[i]; i++)
+ folio_wait_writeback(folios[i]);
+
+ ASSERT(folios[0]);
+ btrfs_lock_extent(&inode->io_tree, folio_pos(folios[0]), cur - 1,
+ &cached_state);
+ /* Still fully mapped to the recorded location? */
+ em = btrfs_get_extent(inode, NULL, start, len);
+ if (IS_ERR(em)) {
+ ret = PTR_ERR(em);
+ goto unlock_extent;
+ }
+ if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE ||
+ (em->flags & EXTENT_FLAG_PREALLOC) ||
+ btrfs_extent_map_compression(em) != BTRFS_COMPRESS_NONE ||
+ em->start > start ||
+ em->start + em->len < start + len ||
+ btrfs_extent_map_block_start(em) + (start - em->start) !=
+ disk_bytenr) {
+ btrfs_free_extent_map(em);
+ goto unlock_extent;
+ }
+ btrfs_free_extent_map(em);
+
+ /*
+ * We hold the folio locks and the extent lock, so a flushing
+ * (ticketed) reservation can deadlock: satisfying the ticket needs
+ * delalloc writeback -- or a transaction commit, which with
+ * flushoncommit waits for that writeback -- and writeback of this
+ * range blocks in __folio_lock on the folios held here. Reserve
+ * with NO_FLUSH; carrying is optional, so on ENOSPC the tail just
+ * stays in its old stripe (see btrfs_carry_log_tail).
+ */
+ ret = btrfs_check_data_free_space(inode, &data_reserved, start, len,
+ true);
+ if (ret < 0)
+ goto unlock_extent;
+ ret = btrfs_delalloc_reserve_metadata(inode, len, len, true);
+ if (ret < 0) {
+ btrfs_free_reserved_data_space(inode, data_reserved, start,
+ len);
+ extent_changeset_free(data_reserved);
+ goto unlock_extent;
+ }
+ btrfs_clear_extent_bit(&inode->io_tree, start, start + len - 1,
+ EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
+ EXTENT_DEFRAG, &cached_state);
+ btrfs_set_extent_bit(&inode->io_tree, start, start + len - 1,
+ EXTENT_DELALLOC, &cached_state);
+ for (int i = 0; i < nr_pages && folios[i]; i++) {
+ struct folio *folio = folios[i];
+
+ if (start >= folio_pos(folio) + folio_size(folio) ||
+ start + len <= folio_pos(folio))
+ continue;
+ btrfs_folio_clamp_set_dirty(fs_info, folio, start, len);
+ }
+ btrfs_delalloc_release_extents(inode, len);
+ extent_changeset_free(data_reserved);
+unlock_extent:
+ btrfs_unlock_extent(&inode->io_tree, folio_pos(folios[0]), cur - 1,
+ &cached_state);
+free_folios:
+ for (int i = 0; i < nr_pages; i++) {
+ if (!folios[i])
+ break;
+ folio_unlock(folios[i]);
+ folio_put(folios[i]);
+ }
+ kfree(folios);
+ return ret;
+}
+
+/*
+ * Consume the inode's carry table (see struct btrfs_log_carry): re-dirty
+ * every recorded range so the caller's upcoming writeback carries them
+ * into the inode's current private stripe run. Called at fsync entry
+ * with the inode locked; takes log_mutex only to detach the table, so
+ * the logging side never records into a half-consumed list. All
+ * failures are soft: an uncarried range just leaves its old stripe to
+ * free the slow way.
+ */
+void btrfs_carry_log_tail(struct btrfs_inode *inode)
+{
+ struct btrfs_log_carry *copy;
+
+ if (!btrfs_test_opt(inode->root->fs_info, STRIPE_ALLOC))
+ return;
+ if (!READ_ONCE(inode->log_carry))
+ return;
+ copy = kmalloc(sizeof(*copy), GFP_NOFS);
+ if (!copy)
+ return;
+ mutex_lock(&inode->log_mutex);
+ if (!inode->log_carry || !inode->log_carry->nr_ranges) {
+ mutex_unlock(&inode->log_mutex);
+ kfree(copy);
+ return;
+ }
+ memcpy(copy, inode->log_carry, sizeof(*copy));
+ inode->log_carry->nr_ranges = 0;
+ mutex_unlock(&inode->log_mutex);
+
+ for (int i = 0; i < copy->nr_ranges; i++)
+ carry_one_range(inode, copy->range[i].start,
+ copy->range[i].len,
+ copy->range[i].disk_bytenr);
+ kfree(copy);
+}
+
static int defrag_one_cluster(struct btrfs_inode *inode,
struct file_ra_state *ra,
u64 start, u32 len, u32 extent_thresh,
int btrfs_defrag_file(struct btrfs_inode *inode, struct file_ra_state *ra,
struct btrfs_ioctl_defrag_range_args *range,
u64 newer_than, unsigned long max_to_defrag);
+void btrfs_carry_log_tail(struct btrfs_inode *inode);
int __init btrfs_auto_defrag_init(void);
void __cold btrfs_auto_defrag_exit(void);
void btrfs_add_inode_defrag(struct btrfs_inode *inode, u32 extent_thresh);
#include "extent-tree.h"
#include "file-item.h"
#include "ioctl.h"
+#include "defrag.h"
#include "file.h"
#include "super.h"
#include "print-tree.h"
atomic_inc(&root->log_batch);
+ /*
+ * Re-dirty the tail ranges recorded at the last log commit so the
+ * writeback below carries them into the inode's current private
+ * stripe run, letting the old partial stripe free whole (see
+ * struct btrfs_log_carry). Purely a space optimization: a range
+ * that cannot be carried is simply skipped.
+ */
+ btrfs_carry_log_tail(inode);
+
/*
* Before we acquired the inode's lock and the mmap lock, someone may
* have dirtied more pages in the target range. We need to make sure
ei->last_reflink_trans = 0;
ei->last_log_commit = 0;
ei->log_run_hint = 0;
+ ei->log_carry = NULL;
spin_lock_init(&ei->lock);
ei->outstanding_extents = 0;
void btrfs_test_destroy_inode(struct inode *inode)
{
btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false);
+ kfree(BTRFS_I(inode)->log_carry);
kfree(BTRFS_I(inode)->file_extent_tree);
kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
}
void btrfs_free_inode(struct inode *inode)
{
+ kfree(BTRFS_I(inode)->log_carry);
kfree(BTRFS_I(inode)->file_extent_tree);
kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
}
disk_num_bytes =
btrfs_file_extent_disk_num_bytes(src, extent);
if (disk_bytenr)
- btrfs_log_settle_stripes(trans->fs_info,
- disk_bytenr,
- disk_num_bytes);
+ btrfs_log_settle_stripes(inode,
+ ins_keys[dst_index].offset,
+ btrfs_file_extent_compression(src, extent) ?
+ 0 :
+ btrfs_file_extent_num_bytes(src, extent),
+ disk_bytenr, disk_num_bytes);
}
if (skip_csum)
*/
if (!(em->flags & EXTENT_FLAG_PREALLOC) &&
em->disk_bytenr < EXTENT_MAP_LAST_BYTE)
- btrfs_log_settle_stripes(inode->root->fs_info, em->disk_bytenr,
- em->disk_num_bytes);
+ btrfs_log_settle_stripes(inode, em->start,
+ compress_type == BTRFS_COMPRESS_NONE ?
+ em->len : 0,
+ em->disk_bytenr, em->disk_num_bytes);
if (compress_type != BTRFS_COMPRESS_NONE) {
btrfs_set_stack_file_extent_disk_bytenr(&fi, block_start);
projects / linux / commitdiff