return div64_u64(val, fsl) * fsl;
}
+static bool stripe_is_aligned(u64 val, u64 fsl)
+{
+ return stripe_round_down(val, fsl) == val;
+}
+
/*
* Band for a run with @remaining bytes of unallocated space: the power-of-two
* bucket of its remaining sectors (band k == remaining in [2^k, 2^(k+1))
return settled;
}
+/*
+ * Remember, on the logging inode, a logged extent that lives in the final
+ * partial stripe of one of the inode's recently closed runs, so that the
+ * next fsync can re-dirty its file range and carry the data forward into
+ * the then-current run -- letting the old partial stripe free whole. A
+ * zero @file_len means the caller does not want this extent carried
+ * (e.g. compressed extents, which would recompress). @ps_len is nonzero
+ * when the settling walk could see the run and computed its final partial
+ * stripe. Every table here is small and bounded; on overflow the extent
+ * simply is not carried and its stripe frees the slow way. Caller holds
+ * the inode's log_mutex, which serializes against the fsync-entry consumer.
+ */
+static void log_carry_record(struct btrfs_inode *inode, u64 file_start,
+ u64 file_len, u64 bytenr, u64 num_bytes,
+ u64 ps_start, u64 ps_len)
+{
+ struct btrfs_log_carry *carry = inode->log_carry;
+ int n, i;
+
+ if (!carry) {
+ carry = kzalloc(sizeof(*carry), GFP_NOFS);
+ if (!carry)
+ return;
+ inode->log_carry = carry;
+ }
+ if (ps_len) {
+ n = min_t(int, carry->nr_ps, BTRFS_LOG_CARRY_NR_PS);
+ for (i = 0; i < n; i++)
+ if (carry->ps_start[i] == ps_start)
+ break;
+ if (i == n) {
+ int slot = carry->nr_ps % BTRFS_LOG_CARRY_NR_PS;
+
+ carry->ps_start[slot] = ps_start;
+ carry->ps_len[slot] = ps_len;
+ carry->nr_ps++;
+ }
+ }
+ if (!file_len || carry->nr_ranges >= BTRFS_LOG_CARRY_NR_RANGES)
+ return;
+ n = min_t(int, carry->nr_ps, BTRFS_LOG_CARRY_NR_PS);
+ for (i = 0; i < n; i++)
+ if (bytenr < carry->ps_start[i] + carry->ps_len[i] &&
+ bytenr + num_bytes > carry->ps_start[i])
+ break;
+ if (i == n)
+ return;
+ for (i = 0; i < carry->nr_ranges; i++)
+ if (carry->range[i].start == file_start &&
+ carry->range[i].len == file_len)
+ return;
+ carry->range[carry->nr_ranges].start = file_start;
+ carry->range[carry->nr_ranges].len = file_len;
+ carry->range[carry->nr_ranges].disk_bytenr = bytenr;
+ carry->nr_ranges++;
+}
+
/*
* Disk end of @inode's last regular file extent, or 0: the re-adoption
* candidate for a nocow allocation arriving with no allocation hint --
* the wait is bio flight time plus the parked-write deadline that the flush
* below short-circuits. It is a pure data-IO wait (no transaction joins),
* safe under the inode log mutex and a running transaction handle.
+ *
+ * [file_start, file_start + file_len) is the extent's file range, recorded
+ * for carry-forward when the extent sits in the run's final partial stripe;
+ * file_len == 0 opts the extent out of carrying (settling still happens).
*/
-void btrfs_log_settle_stripes(struct btrfs_fs_info *fs_info, u64 bytenr,
- u64 num_bytes)
+void btrfs_log_settle_stripes(struct btrfs_inode *inode, u64 file_start,
+ u64 file_len, u64 bytenr, u64 num_bytes)
{
+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct btrfs_block_group *bg;
struct btrfs_open_stripe_run *run;
unsigned long flags;
u64 tail_len = 0;
u64 flush_start = 0;
u64 flush_len = 0;
+ u64 ps_start = 0;
+ u64 ps_len = 0;
/* stripe_alloc_allow_rmw=fsync waives the log-commit guarantee. */
if (btrfs_stripe_allow_rmw(fs_info) & BTRFS_STRIPE_RMW_FSYNC)
spin_lock_irqsave(&bg->stripe_run_lock, flags);
list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ u64 filled;
+
if (bytenr < run->start || bytenr >= run->offset)
continue;
/*
break;
flush_start = run->start;
flush_len = run->end - run->start;
+ /*
+ * The run's final partial stripe, whether this call closes
+ * the run or an earlier settle already did (offset froze at
+ * close). Runs start stripe-aligned, so alignment relative
+ * to run->start is stripe alignment.
+ */
+ filled = run->offset - run->start;
+ if (!stripe_is_aligned(filled, bg->full_stripe_len)) {
+ ps_start = run->start +
+ stripe_round_down(filled, bg->full_stripe_len);
+ ps_len = bg->full_stripe_len;
+ }
if (run->open) {
int class;
int band;
}
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ log_carry_record(inode, file_start, file_len, bytenr, num_bytes,
+ ps_start, ps_len);
+
if (tail_len) {
btrfs_add_free_space(bg, tail_start, tail_len);
stripe_open_remainder_sync(bg);
bool btrfs_stripe_nocow_writable(struct btrfs_inode *inode, u64 start,
u64 num_bytes, u32 which, bool nowait);
int btrfs_parse_stripe_rmw(const char *value, size_t len, u32 *mask);
-void btrfs_log_settle_stripes(struct btrfs_fs_info *fs_info, u64 bytenr,
- u64 num_bytes);
+void btrfs_log_settle_stripes(struct btrfs_inode *inode, u64 file_start,
+ u64 file_len, u64 bytenr, u64 num_bytes);
int btrfs_stripe_alloc_check_support(struct btrfs_fs_info *fs_info);
int btrfs_enable_stripe_alloc(struct btrfs_fs_info *fs_info);
void btrfs_disable_stripe_alloc(struct btrfs_fs_info *fs_info);
BTRFS_INODE_ROOT_STUB,
};
+/*
+ * Carry-forward bookkeeping for a log-active inode whose LOG runs are
+ * private (stripe_log_data_shared clear). ps_start/ps_len remember the
+ * final partial stripes of the inode's recently closed private runs (a
+ * small ring; older entries are overwritten). range[] records the file
+ * ranges of logged extents living in those partial stripes, for the next
+ * fsync to re-dirty; disk_bytenr lets the consumer verify the range still
+ * maps there before carrying. Bounded loss: when the ring or the range
+ * table overflows, the extra extents simply stay put and their stripes free
+ * the slow way.
+ */
+#define BTRFS_LOG_CARRY_NR_PS 4
+#define BTRFS_LOG_CARRY_NR_RANGES 48
+struct btrfs_log_carry {
+ int nr_ps;
+ u64 ps_start[BTRFS_LOG_CARRY_NR_PS];
+ u64 ps_len[BTRFS_LOG_CARRY_NR_PS];
+ int nr_ranges;
+ struct {
+ u64 start;
+ u64 len;
+ u64 disk_bytenr;
+ } range[BTRFS_LOG_CARRY_NR_RANGES];
+};
+
/* in memory btrfs inode */
struct btrfs_inode {
/* which subvolume this inode belongs to */
u64 ref_root_id;
};
+ /*
+ * Carry-forward state (struct btrfs_log_carry), lazily allocated;
+ * recorded and consumed under log_mutex.
+ */
+ struct btrfs_log_carry *log_carry;
+
/*
* Start of the last extent allocated from one of this inode's private
* stripe runs (LOG or NOCOW class). Only a search hint, so
return ret;
}
+/*
+ * Carry-forward for log-active inodes with private LOG runs (see struct
+ * btrfs_log_carry). Re-dirty a file range recorded at the last log commit
+ * so the coming writeback COWs it into the inode's current private stripe
+ * run and the old partial stripe can free whole. The range is carried only
+ * while it still fully maps to the recorded disk bytenr; a range that was
+ * rewritten, truncated, hole-punched or already carried fails that check
+ * and is skipped -- skipping only costs the space optimization, never
+ * correctness. Modeled on defrag_one_range().
+ */
+static int carry_one_range(struct btrfs_inode *inode, u64 start, u64 len,
+ u64 disk_bytenr)
+{
+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
+ struct extent_changeset *data_reserved = NULL;
+ struct extent_state *cached_state = NULL;
+ struct extent_map *em;
+ struct folio **folios;
+ const unsigned int nr_pages = ((start + len - 1) >> PAGE_SHIFT) -
+ (start >> PAGE_SHIFT) + 1;
+ u64 cur;
+ int ret = 0;
+
+ ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
+ IS_ALIGNED(len, fs_info->sectorsize));
+
+ folios = kzalloc_objs(struct folio *, nr_pages, GFP_NOFS);
+ if (!folios)
+ return -ENOMEM;
+
+ cur = start;
+ for (int i = 0; cur < start + len && i < nr_pages; i++) {
+ folios[i] = defrag_prepare_one_folio(inode, cur >> PAGE_SHIFT);
+ if (IS_ERR(folios[i])) {
+ ret = PTR_ERR(folios[i]);
+ folios[i] = NULL;
+ goto free_folios;
+ }
+ cur = folio_next_pos(folios[i]);
+ }
+ for (int i = 0; i < nr_pages && folios[i]; i++)
+ folio_wait_writeback(folios[i]);
+
+ ASSERT(folios[0]);
+ btrfs_lock_extent(&inode->io_tree, folio_pos(folios[0]), cur - 1,
+ &cached_state);
+ /* Still fully mapped to the recorded location? */
+ em = btrfs_get_extent(inode, NULL, start, len);
+ if (IS_ERR(em)) {
+ ret = PTR_ERR(em);
+ goto unlock_extent;
+ }
+ if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE ||
+ (em->flags & EXTENT_FLAG_PREALLOC) ||
+ btrfs_extent_map_compression(em) != BTRFS_COMPRESS_NONE ||
+ em->start > start ||
+ em->start + em->len < start + len ||
+ btrfs_extent_map_block_start(em) + (start - em->start) !=
+ disk_bytenr) {
+ btrfs_free_extent_map(em);
+ goto unlock_extent;
+ }
+ btrfs_free_extent_map(em);
+
+ /*
+ * We hold the folio locks and the extent lock, so a flushing
+ * reservation can deadlock: satisfying a space ticket needs delalloc
+ * writeback -- or a transaction commit, which with flushoncommit waits
+ * for that writeback -- and a qgroup flush (try_flush_qgroup() on
+ * EDQUOT) writes back the root's delalloc and waits for its ordered
+ * extents, or waits for another task doing so. Writeback of this
+ * range blocks in __folio_lock on the folios held here, and the
+ * folios are often dirty: an fsync'ed append dirties the folio holding
+ * the old tail. Reserve space and qgroup space without flushing;
+ * carrying is optional, so on ENOSPC or EDQUOT the tail just stays in
+ * its old stripe (see btrfs_carry_log_tail).
+ */
+ ret = btrfs_check_data_free_space_noflush(inode, &data_reserved, start,
+ len);
+ if (ret < 0)
+ goto unlock_extent;
+ ret = btrfs_delalloc_reserve_metadata(inode, len, len, true);
+ if (ret < 0) {
+ btrfs_free_reserved_data_space(inode, data_reserved, start,
+ len);
+ extent_changeset_free(data_reserved);
+ goto unlock_extent;
+ }
+ /*
+ * Part of the range may already be delalloc (an fsync'ed append
+ * dirties the old tail block). As in btrfs_reset_extent_delalloc(),
+ * clearing it with EXTENT_DO_ACCOUNTING releases that delalloc's own
+ * reservations, and the ones made above cover the whole range. Clear
+ * EXTENT_NORESERVE too: a NOCOW write that reserved no data space must
+ * not leave the flag on a range whose data space is now reserved, or
+ * that reservation is never released if the range is written NOCOW
+ * (the clear hook skips the release for EXTENT_NORESERVE, and checks
+ * the flag before clearing it, so the old state releases nothing).
+ */
+ btrfs_clear_extent_bit(&inode->io_tree, start, start + len - 1,
+ EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
+ EXTENT_DEFRAG | EXTENT_NORESERVE, &cached_state);
+ btrfs_set_extent_bit(&inode->io_tree, start, start + len - 1,
+ EXTENT_DELALLOC, &cached_state);
+ for (int i = 0; i < nr_pages && folios[i]; i++) {
+ struct folio *folio = folios[i];
+
+ if (start >= folio_next_pos(folio) ||
+ start + len <= folio_pos(folio))
+ continue;
+ btrfs_folio_clamp_set_dirty(fs_info, folio, start, len);
+ }
+ btrfs_delalloc_release_extents(inode, len);
+ extent_changeset_free(data_reserved);
+unlock_extent:
+ btrfs_unlock_extent(&inode->io_tree, folio_pos(folios[0]), cur - 1,
+ &cached_state);
+free_folios:
+ for (int i = 0; i < nr_pages; i++) {
+ if (!folios[i])
+ break;
+ folio_unlock(folios[i]);
+ folio_put(folios[i]);
+ }
+ kfree(folios);
+ return ret;
+}
+
+/*
+ * Consume the inode's carry table (see struct btrfs_log_carry): re-dirty
+ * every recorded range so the caller's upcoming writeback carries them
+ * into the inode's current private stripe run. Does nothing with shared
+ * LOG runs (stripe_log_data_shared). Called at fsync entry with the inode
+ * locked; takes log_mutex only to detach the table, so the logging side
+ * never records into a half-consumed list. All failures are soft: an
+ * uncarried range just leaves its old stripe to free the slow way.
+ */
+void btrfs_carry_log_tail(struct btrfs_inode *inode)
+{
+ struct btrfs_log_carry *copy;
+
+ if (!btrfs_test_opt(inode->root->fs_info, STRIPE_ALLOC))
+ return;
+ if (!READ_ONCE(inode->log_carry))
+ return;
+ copy = kmalloc(sizeof(*copy), GFP_NOFS);
+ if (!copy)
+ return;
+ mutex_lock(&inode->log_mutex);
+ if (!inode->log_carry || !inode->log_carry->nr_ranges) {
+ mutex_unlock(&inode->log_mutex);
+ kfree(copy);
+ return;
+ }
+ memcpy(copy, inode->log_carry, sizeof(*copy));
+ inode->log_carry->nr_ranges = 0;
+ mutex_unlock(&inode->log_mutex);
+
+ for (int i = 0; i < copy->nr_ranges; i++)
+ carry_one_range(inode, copy->range[i].start,
+ copy->range[i].len,
+ copy->range[i].disk_bytenr);
+ kfree(copy);
+}
+
static int defrag_one_cluster(struct btrfs_inode *inode,
struct file_ra_state *ra,
u64 start, u32 len, u32 extent_thresh,
int btrfs_defrag_file(struct btrfs_inode *inode, struct file_ra_state *ra,
struct btrfs_ioctl_defrag_range_args *range,
u64 newer_than, unsigned long max_to_defrag);
+void btrfs_carry_log_tail(struct btrfs_inode *inode);
int __init btrfs_auto_defrag_init(void);
void __cold btrfs_auto_defrag_exit(void);
void btrfs_add_inode_defrag(struct btrfs_inode *inode, u32 extent_thresh);
flush, stripe_held);
}
-int btrfs_check_data_free_space(struct btrfs_inode *inode,
- struct extent_changeset **reserved, u64 start,
- u64 len, bool noflush, u64 *stripe_held)
+static int check_data_free_space(struct btrfs_inode *inode,
+ struct extent_changeset **reserved, u64 start,
+ u64 len, bool noflush, bool qgroup_noflush,
+ u64 *stripe_held)
{
struct btrfs_fs_info *fs_info = inode->root->fs_info;
enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_DATA;
*stripe_held = stripe_margin;
/* Use new btrfs_qgroup_reserve_data to reserve precious data space. */
- ret = btrfs_qgroup_reserve_data(inode, reserved, start, len);
+ if (qgroup_noflush)
+ ret = btrfs_qgroup_reserve_data_noflush(inode, reserved, start,
+ len);
+ else
+ ret = btrfs_qgroup_reserve_data(inode, reserved, start, len);
if (ret < 0) {
if (stripe_held) {
btrfs_release_data_margin_probe(inode, stripe_margin);
return ret;
}
+int btrfs_check_data_free_space(struct btrfs_inode *inode,
+ struct extent_changeset **reserved, u64 start,
+ u64 len, bool noflush, u64 *stripe_held)
+{
+ return check_data_free_space(inode, reserved, start, len, noflush,
+ false, stripe_held);
+}
+
+/*
+ * Reserve data space and qgroup space for [@start, @start + @len) without
+ * flushing either, for callers holding folio or extent locks of @inode:
+ * both flushes write back delalloc, the inode's own included. (The noflush
+ * of btrfs_check_data_free_space() covers only the space reservation.)
+ */
+int btrfs_check_data_free_space_noflush(struct btrfs_inode *inode,
+ struct extent_changeset **reserved,
+ u64 start, u64 len)
+{
+ return check_data_free_space(inode, reserved, start, len, true, true,
+ NULL);
+}
/* Release the probe margin btrfs_check_data_free_space() handed back in @stripe_held. */
void btrfs_release_data_margin_probe(struct btrfs_inode *inode, u64 held)
if (held)
btrfs_space_info_free_bytes_may_use(data_sinfo_for_inode(inode), held);
}
+
/*
* Called if we need to clear a data reservation for this inode
* Normally in a error case.
int btrfs_check_data_free_space(struct btrfs_inode *inode,
struct extent_changeset **reserved, u64 start, u64 len,
bool noflush, u64 *stripe_held);
+int btrfs_check_data_free_space_noflush(struct btrfs_inode *inode,
+ struct extent_changeset **reserved,
+ u64 start, u64 len);
void btrfs_release_data_margin_probe(struct btrfs_inode *inode, u64 held);
void btrfs_free_reserved_data_space(struct btrfs_inode *inode,
struct extent_changeset *reserved, u64 start, u64 len);
#include "extent-tree.h"
#include "file-item.h"
#include "ioctl.h"
+#include "defrag.h"
#include "raid56.h"
#include "file.h"
#include "super.h"
else
btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
+ /*
+ * With private LOG runs (stripe_log_data_shared clear), re-dirty the
+ * tail ranges recorded at the last log commit so the writeback below
+ * carries them into the inode's current private stripe run, letting
+ * the old partial stripe free whole (see struct btrfs_log_carry).
+ * Purely a space optimization: a range that cannot be carried is
+ * simply skipped.
+ */
+ btrfs_carry_log_tail(inode);
+
/*
* Before we acquired the inode's lock and the mmap lock, someone may
* have dirtied more pages in the target range. We need to make sure
ei->last_reflink_trans = 0;
ei->last_log_commit = 0;
ei->log_run_hint = 0;
+ ei->log_carry = NULL;
spin_lock_init(&ei->lock);
ei->outstanding_extents = 0;
void btrfs_test_destroy_inode(struct inode *inode)
{
btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false);
+ kfree(BTRFS_I(inode)->log_carry);
kfree(BTRFS_I(inode)->file_extent_tree);
kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
}
void btrfs_free_inode(struct inode *inode)
{
+ kfree(BTRFS_I(inode)->log_carry);
kfree(BTRFS_I(inode)->file_extent_tree);
kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
}
return qgroup_reserve_data(inode, reserved_ret, start, len);
}
+/*
+ * Like btrfs_qgroup_reserve_data(), but return -EDQUOT instead of flushing,
+ * for callers holding folio or extent locks of @inode: the flush writes back
+ * delalloc (the inode's own included) and waits for ordered extents.
+ */
+int btrfs_qgroup_reserve_data_noflush(struct btrfs_inode *inode,
+ struct extent_changeset **reserved_ret,
+ u64 start, u64 len)
+{
+ return qgroup_reserve_data(inode, reserved_ret, start, len);
+}
+
/* Free ranges specified by @reserved, normally in error path */
static int qgroup_free_reserved_data(struct btrfs_inode *inode,
struct extent_changeset *reserved,
/* New io_tree based accurate qgroup reserve API */
int btrfs_qgroup_reserve_data(struct btrfs_inode *inode,
struct extent_changeset **reserved, u64 start, u64 len);
+int btrfs_qgroup_reserve_data_noflush(struct btrfs_inode *inode,
+ struct extent_changeset **reserved,
+ u64 start, u64 len);
int btrfs_qgroup_release_data(struct btrfs_inode *inode, u64 start, u64 len, u64 *released);
int btrfs_qgroup_free_data(struct btrfs_inode *inode,
struct extent_changeset *reserved, u64 start,
disk_num_bytes =
btrfs_file_extent_disk_num_bytes(src, extent);
if (disk_bytenr)
- btrfs_log_settle_stripes(trans->fs_info,
- disk_bytenr,
- disk_num_bytes);
+ btrfs_log_settle_stripes(inode,
+ ins_keys[dst_index].offset,
+ btrfs_file_extent_compression(src, extent) ?
+ 0 :
+ btrfs_file_extent_num_bytes(src, extent),
+ disk_bytenr, disk_num_bytes);
}
if (skip_csum)
*/
if (!(em->flags & EXTENT_FLAG_PREALLOC) &&
em->disk_bytenr < EXTENT_MAP_LAST_BYTE)
- btrfs_log_settle_stripes(inode->root->fs_info, em->disk_bytenr,
- em->disk_num_bytes);
+ btrfs_log_settle_stripes(inode, em->start,
+ compress_type == BTRFS_COMPRESS_NONE ?
+ em->len : 0,
+ em->disk_bytenr, em->disk_num_bytes);
if (compress_type != BTRFS_COMPRESS_NONE) {
btrfs_set_stack_file_extent_disk_bytenr(&fi, block_start);