* group has no runs left.
*/
+static bool stripe_extents_owned_by(struct btrfs_fs_info *fs_info,
+ u64 root_id, u64 ino, u64 start, u64 len,
+ u64 *frontier);
+static bool stripe_range_run_free(struct btrfs_block_group *bg, u64 start,
+ u64 len);
+
struct btrfs_open_stripe_run {
struct list_head list; /* bg->open_stripe_runs */
struct btrfs_block_group *bg;
* Returns 0 and sets *ret_offset, -ENOSPC if no fully-free stripe run in
* this block group can satisfy the allocation, or -ENOMEM.
*/
-int btrfs_alloc_from_inode_stripe_run(struct btrfs_block_group *bg, u64 ino,
+int btrfs_alloc_from_inode_stripe_run(struct btrfs_block_group *bg,
+ struct btrfs_inode *inode,
enum btrfs_stripe_run_class class,
- u64 num_bytes, u64 *ret_offset,
- u64 *available)
+ u64 num_bytes, u64 readopt_hint,
+ u64 *ret_offset, u64 *available)
{
struct btrfs_fs_info *fs_info = bg->fs_info;
const u64 fsl = bg->full_stripe_len;
+ const u64 ino = btrfs_ino(inode);
struct btrfs_open_stripe_run *new_run;
struct btrfs_open_stripe_run *run;
unsigned long flags;
if (!new_run)
return -ENOMEM;
+ /*
+ * Re-adoption (nocow only): after a remount (or eviction) the
+ * inode's last partial stripe has no run, and a fresh claim would
+ * abandon its free tail forever. When the allocation hint points
+ * into a stripe wholly owned by this inode, adopt the tail as the
+ * private run instead: every byte below the frontier is provably
+ * the inode's own committed data, and nothing else can consume a
+ * partial stripe's free space (see btrfs_claim_stripe_tail()).
+ */
+ if (class == BTRFS_STRIPE_RUN_NOCOW && readopt_hint &&
+ readopt_hint >= bg->start &&
+ readopt_hint < bg->start + bg->length) {
+ u64 stripe = bg->start +
+ round_down(readopt_hint - bg->start, fsl);
+ u64 frontier = 0;
+ u64 tlen;
+
+ if (stripe_extents_owned_by(fs_info, btrfs_root_id(inode->root),
+ ino, stripe, fsl, &frontier) &&
+ frontier > stripe && frontier < stripe + fsl &&
+ num_bytes <= stripe + fsl - frontier) {
+ tlen = stripe + fsl - frontier;
+ /*
+ * btrfs_stripe_run_range_usable() is for whole-stripe
+ * claims (it rejects anything shorter than a full
+ * stripe); a tail needs only the run-overlap check.
+ */
+ if (stripe_range_run_free(bg, frontier, tlen) &&
+ !btrfs_claim_stripe_tail(bg, frontier, tlen)) {
+ start = frontier;
+ len = tlen;
+ goto install;
+ }
+ }
+ }
+
ret = btrfs_claim_free_stripe_run(bg,
div64_u64(num_bytes + fsl - 1, fsl) * fsl,
&start, &len);
goto out;
}
+install:
/* See the installation comment in btrfs_alloc_from_open_stripe(). */
spin_lock(&fs_info->open_stripe_lock);
if (list_empty(&bg->open_stripe_bg_list)) {
* the whole stripe frees and commits -- which the extent tree shows.
*/
static bool stripe_extents_owned_by(struct btrfs_fs_info *fs_info,
- u64 root_id, u64 ino, u64 start, u64 len)
+ u64 root_id, u64 ino, u64 start, u64 len,
+ u64 *frontier)
{
struct btrfs_root *extent_root = btrfs_extent_root(fs_info, start);
struct btrfs_path *path;
ret = false;
goto out;
}
+ if (frontier && key.objectid + key.offset > *frontier)
+ *frontier = min(key.objectid + key.offset, start + len);
path->slots[0]++;
}
out:
spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
if (verdict == 0 &&
- !stripe_extents_owned_by(fs_info, root_id, ino, cur, fsl))
+ !stripe_extents_owned_by(fs_info, root_id, ino, cur, fsl,
+ NULL))
verdict = -1;
if (verdict < 0) {
ret = false;
carry->nr_ranges++;
}
+/*
+ * Disk end of @inode's last regular file extent, or 0: the re-adoption
+ * candidate for a nocow allocation arriving with no allocation hint --
+ * after a remount or eviction the extent maps are gone, so the normal
+ * em-based hint is empty exactly when re-adoption matters most.
+ */
+u64 btrfs_nocow_alloc_hint(struct btrfs_inode *inode)
+{
+ struct btrfs_root *root = inode->root;
+ struct btrfs_path *path;
+ struct btrfs_key key;
+ struct btrfs_file_extent_item *fi;
+ u64 hint = 0;
+ int ret;
+
+ path = btrfs_alloc_path();
+ if (!path)
+ return 0;
+ key.objectid = btrfs_ino(inode);
+ key.type = BTRFS_EXTENT_DATA_KEY;
+ key.offset = (u64)-1;
+ ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
+ if (ret < 0)
+ goto out;
+ ret = btrfs_previous_item(root, path, key.objectid,
+ BTRFS_EXTENT_DATA_KEY);
+ if (ret)
+ goto out;
+ btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
+ fi = btrfs_item_ptr(path->nodes[0], path->slots[0],
+ struct btrfs_file_extent_item);
+ if (btrfs_file_extent_type(path->nodes[0], fi) ==
+ BTRFS_FILE_EXTENT_INLINE)
+ goto out;
+ if (btrfs_file_extent_disk_bytenr(path->nodes[0], fi))
+ hint = btrfs_file_extent_disk_bytenr(path->nodes[0], fi) +
+ btrfs_file_extent_disk_num_bytes(path->nodes[0], fi);
+out:
+ btrfs_free_path(path);
+ return hint;
+}
+
+/* Does no run object (open or draining) overlap [start, start + len)? */
+static bool stripe_range_run_free(struct btrfs_block_group *bg, u64 start,
+ u64 len)
+{
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ bool ret = true;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (run->start < start + len && run->end > start) {
+ ret = false;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ return ret;
+}
+
+/*
+ * Close every private stripe run owned by @inode, of any class: called at
+ * inode eviction so a persistent nocow run's claimed tail does not stay
+ * unusable for the rest of the mount (log-class runs would close at the
+ * next commit anyway; closing them here is merely tidy). The runs' tails
+ * return to the free space cache.
+ */
+void btrfs_close_inode_stripe_runs(struct btrfs_inode *inode)
+{
+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
+ const u64 ino = btrfs_ino(inode);
+ struct btrfs_block_group *bg;
+
+ if (list_empty_careful(&fs_info->open_stripe_bgs))
+ return;
+restart:
+ spin_lock(&fs_info->open_stripe_lock);
+ list_for_each_entry(bg, &fs_info->open_stripe_bgs,
+ open_stripe_bg_list) {
+ struct btrfs_open_stripe_run *run;
+ unsigned long flags;
+ u64 tail_start = 0;
+ u64 tail_len = 0;
+
+ spin_lock_irqsave(&bg->stripe_run_lock, flags);
+ list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ if (!run->open || run->owner != ino)
+ continue;
+ tail_len = close_open_stripe_run(bg, run, &tail_start);
+ break;
+ }
+ spin_unlock_irqrestore(&bg->stripe_run_lock, flags);
+ if (tail_len) {
+ btrfs_get_block_group(bg);
+ spin_unlock(&fs_info->open_stripe_lock);
+ btrfs_add_free_space(bg, tail_start, tail_len);
+ btrfs_put_block_group(bg);
+ goto restart;
+ }
+ }
+ spin_unlock(&fs_info->open_stripe_lock);
+}
+
/*
* A log commit is about to make the extent at [bytenr, bytenr + num_bytes)
* durable. Once the log super lands, no later write may extend or RMW the
if (!run->open || run->open_seq >= seq)
continue;
+ /*
+ * A nodatacow inode's private run survives transaction
+ * commits: its stripes hold only that inode's expendable
+ * (write-hole-waived) data, so invariant I2 has nothing to
+ * protect there, and closing would burn a stripe per commit
+ * for a slowly appended nocow file. Forced quiescing
+ * (read-only, removal, unmount: seq == U64_MAX) still closes
+ * them, as does the owning inode's eviction.
+ */
+ if (run->class == BTRFS_STRIPE_RUN_NOCOW && seq != U64_MAX)
+ continue;
for (class = 0; class < BTRFS_STRIPE_RUN_NR_CLASSES; class++)
for (band = 0; band < BTRFS_STRIPE_RUN_NR_BANDS; band++)
if (bg->open_stripe[class][band] == run)
spin_lock_irqsave(&bg->stripe_run_lock, flags);
list_for_each_entry(run, &bg->open_stripe_runs, list) {
+ /* Persistent nocow runs are not part of any commit window. */
+ if (run->class == BTRFS_STRIPE_RUN_NOCOW && seq != U64_MAX)
+ continue;
if (run->open_seq < seq) {
ret = false;
break;
* Returns 0 on success, -ENOSPC if no aligned fully-free stripe is present
* in this block group.
*/
+/*
+ * Claim exactly [start, start + len) from the free space cache, failing
+ * (-ENOSPC) unless every byte of it is currently free. Used to re-adopt
+ * the free tail of a partial stripe wholly owned by one nodatacow inode:
+ * nothing else can consume free space inside a partially used stripe of a
+ * stripe_alloc block group (allocations come only from claimed runs, and
+ * runs claim only fully-free stripes), so the verify-then-remove sequence
+ * cannot race with another consumer -- except a concurrent re-adoption by
+ * the same inode, which this function's recount safely turns into one
+ * winner and one -ENOSPC.
+ */
+int btrfs_claim_stripe_tail(struct btrfs_block_group *block_group,
+ u64 start, u64 len)
+{
+ struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
+ const u64 end = start + len;
+ u64 found = 0;
+ struct rb_node *n;
+
+ spin_lock(&ctl->tree_lock);
+ for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) {
+ struct btrfs_free_space *e =
+ rb_entry(n, struct btrfs_free_space, offset_index);
+ u64 e_start = e->offset;
+ u64 e_end;
+
+ if (e->bitmap) {
+ const u64 unit = block_group->fs_info->sectorsize;
+ unsigned long b = 0;
+
+ e_end = e_start + (u64)BITS_PER_BITMAP * unit;
+ if (e_end <= start || e_start >= end)
+ continue;
+ while (b < BITS_PER_BITMAP) {
+ unsigned long z;
+ u64 f_start, f_end;
+
+ b = find_next_bit(e->bitmap, BITS_PER_BITMAP, b);
+ if (b >= BITS_PER_BITMAP)
+ break;
+ z = find_next_zero_bit(e->bitmap,
+ BITS_PER_BITMAP, b);
+ f_start = max(start, e_start + (u64)b * unit);
+ f_end = min(end, e_start + (u64)z * unit);
+ if (f_end > f_start)
+ found += f_end - f_start;
+ b = z;
+ }
+ continue;
+ }
+ e_end = e_start + e->bytes;
+ if (e_end <= start || e_start >= end)
+ continue;
+ found += min(end, e_end) - max(start, e_start);
+ }
+ spin_unlock(&ctl->tree_lock);
+
+ if (found != len)
+ return -ENOSPC;
+ return btrfs_remove_free_space(block_group, start, len);
+}
+
int btrfs_claim_free_stripe_run(struct btrfs_block_group *block_group,
u64 want_bytes, u64 *start, u64 *len)
{
projects / linux / commitdiff