From 6c0b211fe098114fe80cabddcab74351ecfa030d Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Fri, 4 Sep 2026 09:41:48 -0700 Subject: [PATCH] btrfs: zstd: avoid a copy in zstd_decompress_bio() zstd_decompress_bio() gives zstd a sectorsize-sized scratch buffer, and btrfs_decompress_buf2page() then copies the part overlapping the read bio into the destination folios. Every delivered byte is written twice. Instead, choose the output buffer per streaming call. zstd_map_dest() kmaps the current page-bounded segment of the read bio, so zstd writes into the page cache directly. The scratch buffer is kept only for output with no destination: the prefix before a read starting inside a compressed extent, which zstd cannot skip, and gaps left by folios already in the page cache. Varying the output buffer across calls is safe: btrfs uses the default ZSTD_bm_buffered mode, where the sliding window lives in the dstream's internal buffer and the caller's dst is a pure sink. The read bio's iterator must still advance by exactly the bytes delivered, since btrfs_decompress_bio() zero-fills from it; that used to happen inside btrfs_decompress_buf2page() and is now an explicit bio_advance(), made only for output that reached a folio. bio_iter_iovec() exposes at most one base page, so direct output is page-bounded. Compared to the old sectorsize-sized chunks, this can increase stream calls when sectorsize exceeds PAGE_SIZE, but eliminates the extra btrfs copy for output delivered to the read bio; the 64 KiB sectorsize row below shows the copy still wins there. Benchmarked the change in 2-vCPU x86-64 KVM guests (4 KiB pages, RAM disk) using a 64 MiB zstd-compressed file. Results are medians of seven cold-cache reads in each of six interleaved A/B boot pairs; mincore confirmed zero resident pages before every run. Normal sequential reads with readahead produced: sectorsize base patched reduction 4 KiB 8.678 ms 8.004 ms 7.80% 16 KiB 8.216 ms 7.934 ms 3.64% 64 KiB 7.875 ms 7.344 ms 6.88% Random 4 KiB preads at 4 KiB sectorsize, means of six interleaved A/B boot pairs, patched better in all six: base patched gain 264.33 MB/s 272.67 MB/s 3.2% Signed-off-by: Usama Arif Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/zstd.c | 69 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c index 58d9ff76fe07b..280ac5273438b 100644 --- a/fs/btrfs/zstd.c +++ b/fs/btrfs/zstd.c @@ -589,10 +589,48 @@ out: return ret; } +/* + * Map the destination for the next chunk of output. + * + * @decompressed is the offset of the next output byte inside the fully + * decompressed extent. If that offset has reached the current destination + * segment, its page-bounded bio_vec is kmapped so that zstd can write into the + * page cache directly, and the number of bytes writable there is returned. + * Otherwise @kaddr_ret is set to NULL and the number of bytes to skip before + * that segment is returned. This covers both the initial prefix and gaps in + * the destination bio. + */ +static u32 zstd_map_dest(struct compressed_bio *cb, u32 decompressed, + void **kaddr_ret) +{ + struct bio *orig_bio = &cb->orig_bbio->bio; + struct bio_vec bvec; + u32 bvec_offset; + u32 off; + + bvec = bio_iter_iovec(orig_bio, orig_bio->bi_iter); + /* + * cb->start may underflow, but subtracting that value can still give us + * the correct offset inside the full decompressed extent. + */ + bvec_offset = page_offset(bvec.bv_page) + bvec.bv_offset - cb->start; + + if (decompressed < bvec_offset) { + *kaddr_ret = NULL; + return bvec_offset - decompressed; + } + + off = decompressed - bvec_offset; + ASSERT(off < bvec.bv_len); + *kaddr_ret = bvec_kmap_local(&bvec) + off; + return bvec.bv_len - off; +} + int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) { struct btrfs_fs_info *fs_info = cb_to_fs_info(cb); struct workspace *workspace = list_entry(ws, struct workspace, list); + struct bio *orig_bio = &cb->orig_bbio->bio; struct folio_iter fi; size_t srclen = bio_get_size(&cb->bbio.bio); zstd_dstream *stream; @@ -600,7 +638,6 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) const unsigned int min_folio_size = btrfs_min_folio_size(fs_info); unsigned long folio_in_index = 0; unsigned long total_folios_in = DIV_ROUND_UP(srclen, min_folio_size); - unsigned long buf_start; unsigned long total_out = 0; bio_first_folio(&fi, &cb->bbio.bio, 0); @@ -624,15 +661,26 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) workspace->in_buf.pos = 0; workspace->in_buf.size = min_t(size_t, srclen, min_folio_size); - workspace->out_buf.dst = workspace->buf; - workspace->out_buf.pos = 0; - workspace->out_buf.size = fs_info->sectorsize; - - while (1) { + while (orig_bio->bi_iter.bi_size) { size_t ret2; + void *kaddr; + u32 dstlen; + + dstlen = zstd_map_dest(cb, total_out, &kaddr); + if (kaddr) { + workspace->out_buf.dst = kaddr; + workspace->out_buf.size = dstlen; + } else { + workspace->out_buf.dst = workspace->buf; + workspace->out_buf.size = min_t(u32, dstlen, + fs_info->sectorsize); + } + workspace->out_buf.pos = 0; ret2 = zstd_decompress_stream(stream, &workspace->out_buf, &workspace->in_buf); + if (kaddr) + kunmap_local(kaddr); if (unlikely(zstd_is_error(ret2))) { struct btrfs_inode *inode = cb->bbio.inode; @@ -643,14 +691,9 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) ret = -EIO; goto done; } - buf_start = total_out; total_out += workspace->out_buf.pos; - workspace->out_buf.pos = 0; - - ret = btrfs_decompress_buf2page(workspace->out_buf.dst, - total_out - buf_start, cb, buf_start); - if (ret == 0) - break; + if (kaddr) + bio_advance(orig_bio, workspace->out_buf.pos); if (workspace->in_buf.pos >= srclen) break; -- 2.53.0