const uint64_t old_bucket_count = old_size / BLOCK_SIZE_HASHTAB_BUCKET;
const bool no_merge = old_bucket_count > 0 && (new_buckets % old_bucket_count) == 0;
+ // The other divisible case is the shrink: new_buckets divides
+ // old_bucket_count, so exactly merge_width old buckets feed each new
+ // bucket and merging is unavoidable. Merging does not require the
+ // c_cells_per_bucket passes below, though -- those exist only because
+ // that loop visits the old table in file order, and a cell's LRU rank
+ // is its position in its old bucket, which stops being recoverable
+ // once two old buckets have been interleaved into one new bucket.
+ //
+ // Visiting in destination order instead keeps every rank readable
+ // straight off the array index: gather the merge_width old buckets
+ // that feed one new bucket, then merge them by position. Nothing has
+ // to be stored per cell -- the rank was never anywhere but the index.
+ //
+ // The gather is whole extents, not scattered buckets. New bucket n is
+ // fed by old buckets n + j * new_buckets, and new_buckets is a
+ // multiple of buckets_per_extent because new_size is a multiple of one
+ // Extent, so new extent e is fed by old extents e + j * new_extents --
+ // merge_width whole extents, and every old extent is read by exactly
+ // one new extent. One pass, all of it sequential within an extent.
+ //
+ // The buffer is merge_width extents, so cap it: a ratio past this
+ // falls back to the loop below rather than allocating without bound.
+ // 1024 extents is 128 MiB, which is a 1024:1 shrink -- far past any
+ // resize an operator would ask for.
+ static const uint64_t c_resize_gather_max_extents = 1024;
+ const uint64_t merge_width = (new_buckets > 0 && (old_bucket_count % new_buckets) == 0)
+ ? old_bucket_count / new_buckets : 0;
+ const bool gather_merge = !no_merge
+ && merge_width > 1
+ && merge_width <= c_resize_gather_max_extents
+ // The extent arithmetic above needs the old table to tile exactly
+ // into Extents, the same property open_file() guarantees for the
+ // new one. Anything else keeps the cell-pass loop.
+ && (old_size % BLOCK_SIZE_HASHTAB_EXTENT) == 0;
+
// Cell-position is the OUTER loop, not the inner. When two old
// buckets collide on one new bucket (the common case on shrink:
// cutting buckets in half forces every cell to share a new bucket
// new bucket retains the newest cells across all colliding old
// buckets — which is the LRU semantics we want.
//
- // Cost when merging: c_cells_per_bucket × old_extents pread calls
+ // This loop now runs only when neither divisible case applies, i.e.
+ // old_buckets and new_buckets do not divide each other in either
+ // direction. Cost: c_cells_per_bucket × old_extents pread calls
// instead of old_extents. When the old file fits in the page cache
// this is fast; when it does not, every pass after the first reads
// from disk again.
// BeesProgressLogger meaningful — as long as the page cache holds the
// old table. Once it does not, the estimate runs optimistic because
// later passes fault in from disk.
- const uint64_t total_reads = no_merge ? old_extents : c_cells_per_bucket * old_extents;
+ const uint64_t total_reads = (no_merge || gather_merge) ? old_extents : c_cells_per_bucket * old_extents;
BeesProgressLogger resize_progress("Resizing hash table");
ByteVector extent_buf(BLOCK_SIZE_HASHTAB_EXTENT);
if (no_merge) {
}
}
}
+ } else if (gather_merge) {
+ BEESLOGINFO("Resizing with a " << merge_width << "-way destination-major merge: "
+ << new_buckets << " new buckets divide " << old_bucket_count
+ << " old buckets, one pass");
+ ByteVector gather_buf(merge_width * BLOCK_SIZE_HASHTAB_EXTENT);
+ for (uint64_t e = 0; e < new_extents; ++e) {
+ // Gather the merge_width old extents that feed new extent e.
+ for (uint64_t j = 0; j < merge_width; ++j) {
+ const uint64_t ei = e + j * new_extents;
+ const off_t offset = ranged_cast<off_t>(ei) * BLOCK_SIZE_HASHTAB_EXTENT;
+ pread_or_die(m_fd, gather_buf.data() + j * BLOCK_SIZE_HASHTAB_EXTENT,
+ BLOCK_SIZE_HASHTAB_EXTENT, offset);
+ resize_progress.update(e * merge_width + j + 1, total_reads);
+ }
+
+ // Merge the gathered buckets into the new bucket they share.
+ // The push order is the same subsequence the cell-pass loop
+ // below would produce for this new bucket -- LRU-oldest rank
+ // first so the newest ends up at the front, and within a rank,
+ // descending old extent index -- so the output is bit-identical
+ // for any table whose cells are in the bucket their hash says.
+ // Old bucket n + j * new_buckets sits at the same offset b
+ // inside its extent as new bucket n does, because new_buckets
+ // is a whole number of extents, so one b indexes all of them.
+ for (uint64_t b = 0; b < buckets_per_extent; ++b) {
+ for (uint64_t ci = c_cells_per_bucket; ci > 0; --ci) {
+ for (uint64_t jr = merge_width; jr > 0; --jr) {
+ const Bucket *const old_bucket_ptr = reinterpret_cast<const Bucket *>(
+ gather_buf.data() + (jr - 1) * BLOCK_SIZE_HASHTAB_EXTENT);
+ const Cell &c = old_bucket_ptr[b].p_cells[ci - 1];
+ if (!c.e_addr) continue;
+ // Still addressed by hash, not by the b above: a cell
+ // that is not in the bucket its hash names (a damaged
+ // or foreign table) then still lands in the right new
+ // bucket, just not necessarily in the order the
+ // cell-pass loop would have put it in.
+ Bucket &dst = new_bucket_ptr[c.e_hash % new_buckets];
+ push_front_in_range(dst.p_cells, dst.p_cells + c_cells_per_bucket, c);
+ }
+ }
+ }
+ }
} else {
for (uint64_t ci = c_cells_per_bucket; ci > 0; --ci) {
const uint64_t cell_idx = ci - 1;
projects / bees / commitdiff