BEESTRACE("hash table bucket size " << BLOCK_SIZE_HASHTAB_BUCKET);
BEESTRACE("hash table extent size " << BLOCK_SIZE_HASHTAB_EXTENT);
- BEESLOGINFO("opened hash table filename '" << m_filename << "' length " << m_size);
+ // m_size is the size the table will have when prepare() is done, which
+ // during a resize is the requested size, not the current file's.
+ BEESLOGINFO("mapping hash table '" << m_filename << "' length " << m_size << " (" << pretty(m_size) << ")");
m_buckets = m_size / BLOCK_SIZE_HASHTAB_BUCKET;
m_cells = m_buckets * c_cells_per_bucket;
m_extents = (m_size + BLOCK_SIZE_HASHTAB_EXTENT - 1) / BLOCK_SIZE_HASHTAB_EXTENT;
return;
}
- // Apply state.hash.resize before mmap so the mapping is the size
- // the operator requested, not the current on-disk size. When a
- // resize is in progress this is what keeps the host from having to
- // reserve the OLD size of virtual address space (or commit it via
- // mlock) on a machine that may have been downsized.
+ // Decide whether this startup is a resize before anything is mapped.
+ // open_file() left m_size at the on-disk size; m_desired_size is what
+ // [state.hash] size asked for.
+ const uint64_t old_size = m_size;
+ bool do_resize = false;
if (m_persistent && m_size != m_desired_size) {
if (m_resize) {
- resize_file(ranged_cast<off_t>(m_desired_size));
+ do_resize = true;
+ // Map at the size the operator requested, not the on-disk
+ // size. On a host that was downsized the old size may not
+ // be mappable at all, and it is never what we want resident.
+ m_size = m_desired_size;
} else {
BEESLOGWARN("hash table on-disk size " << pretty(m_size)
<< " differs from configured state.hash.size " << pretty(m_desired_size)
}
}
- // mmap + size metadata at the final size.
+ // Map and commit the final-size table before doing anything with it.
+ // A resize then fills this mapping in place, so it inherits
+ // MADV_HUGEPAGE and the mlock. Building the new table in a heap
+ // buffer instead left multiple gigabytes unlocked and un-advised while
+ // the resize scattered writes across all of it and streamed the old
+ // file through the page cache, and the host swapped heavily.
mmap_and_size_metadata();
+ mlock_mapping();
- // Load the whole table into the (anonymous) mapping up front, in the
- // main thread, before any worker can look up a hash. This replaces
- // the old per-extent demand-load: random worker faults turned the
- // read into scattered I/O, so a single sequential pass here is both
- // faster to steady state and lets the lookup/insert paths assume the
- // table is always resident. Memory-only mode skips this —
- // the pages are already zero from MAP_ANONYMOUS.
- BeesProgressLogger load_progress("Loading hash table '" + m_filename + "'");
- for (uint64_t ext = 0; ext < m_extents; ++ext) {
+ if (do_resize) {
+ // Fills the mapping from the old file and persists it. No load
+ // pass afterwards: the mapping already holds the new table, and
+ // the file on disk was written from it.
+ resize_into_mapping(old_size);
+ } else {
+ // Load the whole table into the (anonymous) mapping up front, in
+ // the main thread, before any worker can look up a hash. This
+ // replaces the old per-extent demand-load: random worker faults
+ // turned the read into scattered I/O, so a single sequential pass
+ // here is both faster to steady state and lets the lookup/insert
+ // paths assume the table is always resident. Memory-only mode
+ // skips the read — the pages are already zero from MAP_ANONYMOUS.
+ BeesProgressLogger load_progress("Loading hash table '" + m_filename + "'");
+ for (uint64_t ext = 0; ext < m_extents; ++ext) {
+ if (m_persistent) {
+ uint8_t *const extent_begin = m_extent_ptr[ext ].p_byte;
+ uint8_t *const extent_end = m_extent_ptr[ext + 1].p_byte;
+ pread_or_die(m_fd, extent_begin, extent_end - extent_begin, extent_begin - m_byte_ptr);
+ BEESCOUNT(hash_extent_in);
+ load_progress.update(ext + 1, m_extents);
+ }
+ // Seed the occupancy fragment from the just-loaded (or, in
+ // memory-only mode, zeroed) extent so the survey has data
+ // before the first writeback.
+ recompute_extent_fill_histogram_locked(ext);
+ }
if (m_persistent) {
- uint8_t *const extent_begin = m_extent_ptr[ext ].p_byte;
- uint8_t *const extent_end = m_extent_ptr[ext + 1].p_byte;
- pread_or_die(m_fd, extent_begin, extent_end - extent_begin, extent_begin - m_byte_ptr);
- BEESCOUNT(hash_extent_in);
- load_progress.update(ext + 1, m_extents);
+ load_progress.finish();
}
- // Seed the occupancy fragment from the just-loaded (or, in
- // memory-only mode, zeroed) extent so the survey has data before
- // the first writeback.
- recompute_extent_fill_histogram_locked(ext);
- }
- if (m_persistent) {
- load_progress.finish();
}
+}
- // Pre-fault and lock now, while we know which size the host has to
- // back. prefetch_loop() used to do this on its first analyze pass,
- // but by then the mmap had already been allocated unconditionally
- // at the on-disk size — wrong for resize-down scenarios. Doing
- // allocation + lock together here means a host that cannot back
- // the requested size fails fast and visibly. catch_all to support
- // users who don't want to use mlock().
- if (m_cell_ptr) {
- BEESLOGINFO("mlock(" << pretty(m_size) << ")...");
- Timer lock_time;
- catch_all([&]() {
- DIE_IF_NON_ZERO(mlock(m_byte_ptr, m_size));
- });
- BEESLOGINFO("mlock(" << pretty(m_size) << ") done in " << lock_time << " sec");
+void
+BeesHashTable::mlock_mapping()
+{
+ // Commit the pages now, while we know which size the host has to back.
+ // prefetch_loop() used to do this on its first analyze pass, but by
+ // then the mmap had already been allocated unconditionally at the
+ // on-disk size — wrong for resize-down scenarios. Doing allocation +
+ // lock together here means a host that cannot back the requested size
+ // fails fast and visibly. catch_all to support users who don't want
+ // to use mlock().
+ if (!m_cell_ptr) {
+ return;
}
+ BEESLOGINFO("mlock(" << pretty(m_size) << ")...");
+ Timer lock_time;
+ catch_all([&]() {
+ DIE_IF_NON_ZERO(mlock(m_byte_ptr, m_size));
+ });
+ BEESLOGINFO("mlock(" << pretty(m_size) << ") done in " << lock_time << " sec");
}
void
}
void
-BeesHashTable::resize_file(off_t new_size_signed)
+BeesHashTable::resize_into_mapping(const uint64_t old_size)
{
THROW_CHECK1(runtime_error, m_persistent, m_persistent);
- const uint64_t new_size = ranged_cast<uint64_t>(new_size_signed);
- if (m_size == new_size) {
- return;
- }
+ // prepare() has already mapped, advised, and locked the mapping at the
+ // new size, so the destination is m_size and the union aliases over it
+ // (m_bucket_ptr, m_extent_ptr) are the ones to write through. There
+ // is no second copy of the table anywhere.
+ const uint64_t new_size = m_size;
+ THROW_CHECK2(invalid_argument, old_size, new_size, old_size != new_size);
+ THROW_CHECK1(runtime_error, m_cell_ptr, m_cell_ptr != nullptr);
- const uint64_t old_size = m_size;
const uint64_t old_extents = old_size / BLOCK_SIZE_HASHTAB_EXTENT;
- const uint64_t new_buckets = new_size / BLOCK_SIZE_HASHTAB_BUCKET;
+ const uint64_t new_buckets = m_buckets;
+ const uint64_t new_extents = m_extents;
const uint64_t buckets_per_extent = BLOCK_SIZE_HASHTAB_EXTENT / BLOCK_SIZE_HASHTAB_BUCKET;
BEESLOGINFO("Resizing hash table from " << pretty(old_size) << " (" << old_extents << " extents) to " << pretty(new_size));
- // One full new-table copy in RAM is acceptable here — the OOM case
- // is having OLD and NEW resident at the same time, which we avoid by
- // reading the old file one extent at a time below. When the old
- // table fits in the page cache the rereads land in cache; when it
- // does not the resize is slow but does not OOM.
- //
// open_file() guarantees the table size is an exact multiple of one
- // Extent, so both views below tile the buffer exactly and the write
+ // Extent, so the Extent view tiles the mapping exactly and the write
// loop has no partial tail extent to special-case.
THROW_CHECK2(invalid_argument, new_size, BLOCK_SIZE_HASHTAB_EXTENT, (new_size % BLOCK_SIZE_HASHTAB_EXTENT) == 0);
- const uint64_t new_extents = new_size / BLOCK_SIZE_HASHTAB_EXTENT;
- Timer alloc_timer;
- // Raw bytes, not vector<Extent>: Cell has no default constructor by
- // design, so Bucket and Extent have none either. Both are union
- // aliases over the same bytes, exactly as the mmap'd table is viewed
- // through m_bucket_ptr and m_extent_ptr.
- vector<uint8_t> new_table(new_size, 0);
- Bucket *const new_bucket_ptr = reinterpret_cast<Bucket *>(new_table.data());
- const Extent *const new_extent_ptr = reinterpret_cast<const Extent *>(new_table.data());
- BEESLOGDEBUG("Allocated " << pretty(new_size) << " (" << new_extents << " extents)"
- " for the resized hash table in " << alloc_timer << " sec");
+
+ // The mapping is MAP_ANONYMOUS, so it starts zeroed; the old contents
+ // are read one extent at a time into a scratch buffer below, which is
+ // the only extra memory this uses.
+ Bucket *const new_bucket_ptr = m_bucket_ptr;
// Can two old buckets land in the same new bucket? A cell is placed
// at new_bucket = hash % new_buckets, and lived at
// progress is reportable in — writing 16 GiB to a slow device is
// minutes of otherwise silent time. Durability comes from the fsync
// and rename below, not from the size of any one write.
+ //
+ // Seed each extent's occupancy histogram in the same pass, while the
+ // extent is the one we just touched. prepare() does this inside its
+ // load loop for the non-resize path; doing it here saves a second
+ // sequential walk of the whole mapping.
BeesProgressLogger write_progress("Writing resized hash table to '" + tmp_filename + "'");
for (uint64_t ext = 0; ext < new_extents; ++ext) {
- const Extent &extent = new_extent_ptr[ext];
+ const Extent &extent = m_extent_ptr[ext];
pwrite_or_die(tmp_fd, extent.p_byte, sizeof(extent.p_byte),
ranged_cast<off_t>(ext * sizeof(Extent)));
+ recompute_extent_fill_histogram_locked(ext);
write_progress.update(ext + 1, new_extents);
}
write_progress.finish();
tmp_fd = Fd();
renameat_or_die(m_ctx->home_fd(), tmp_filename, m_ctx->home_fd(), m_filename);
- // Reopen the renamed file and update m_size.
+ // Reopen the renamed file. m_size was already set to the new size by
+ // prepare() before the mapping was created, so there is nothing to
+ // update here; check that the file we just wrote agrees.
m_fd = openat_or_die(m_ctx->home_fd(), m_filename, FLAGS_OPEN_FILE_RW, 0700);
- m_size = new_size;
+ const off_t reopened_size = Stat(m_fd).st_size;
+ THROW_CHECK2(runtime_error, reopened_size, new_size, ranged_cast<uint64_t>(reopened_size) == new_size);
BEESLOGINFO("Hash table resize complete: " << pretty(old_size) << " -> " << pretty(new_size));
}
/// size is known.
void mmap_and_size_metadata();
+ /// mlock the mapping created by mmap_and_size_metadata(), committing
+ /// the pages so neither startup nor steady-state work can swap them
+ /// out. catch_all: a host that declines to lock still runs.
+ void mlock_mapping();
+
public:
/// Apply state.hash.resize (if requested), mmap the in-memory hash
/// table at the final size, mlock it. Idempotent: subsequent calls
/// been mutated to move @p mv to the front).
///
/// This is the lock-free, fetch-free core of push_front_hash_addr.
- /// resize_file() calls it directly to populate a freshly allocated
- /// new-size buffer without dragging in the live mmap's lazy-fetch
- /// path (which would read from the *old* m_fd into the *new* buffer
- /// and corrupt it) or its per-extent locking (single-threaded at
- /// startup, no locking required).
+ /// resize_into_mapping() calls it directly to populate the new-size
+ /// mapping without dragging in the live mmap's lazy-fetch path (which
+ /// would read from the *old* m_fd into the *new* mapping and corrupt
+ /// it) or its per-extent locking (single-threaded at startup, no
+ /// locking required).
static bool push_front_in_range(Cell *begin, Cell *end, const Cell &mv);
void writeback_loop();
pair<uint8_t *, uint8_t *> get_extent_range(HashType hash);
void set_extent_dirty_locked(uint64_t extent_index);
size_t flush_dirty_extents(bool slowly);
- void resize_file(off_t new_size);
+
+ /// Redistribute the cells of the @p old_size hash table file into the
+ /// mapping already created at the new m_size, then persist that
+ /// mapping over the old file (temp file, fsync, atomic rename) and
+ /// reopen m_fd on the result.
+ ///
+ /// Takes the OLD size, not the new one: prepare() has already set
+ /// m_size to the requested size and mapped and locked it, so this only
+ /// needs to know where it is reading from. Filling the real mapping
+ /// is what gives the resize MADV_HUGEPAGE and the mlock; building the
+ /// new table in a heap buffer instead left it swappable, and scattering
+ /// writes across an unlocked multi-gigabyte buffer while streaming the
+ /// old file through the page cache made the host swap heavily.
+ void resize_into_mapping(uint64_t old_size);
size_t hash_to_extent_index(HashType ht);
unique_lock<mutex> lock_extent_by_hash(HashType ht);
projects / bees / commitdiff