size_t toxic_count = 0;
size_t unaligned_eof_count = 0;
- m_prefetch_running = true;
for (uint64_t ext = 0; ext < m_extents && !m_stop_requested; ++ext) {
- BEESNOTE("prefetching hash table extent #" << ext << " of " << m_extents);
catch_all([&]() {
- fetch_missing_extent_by_index(ext);
-
BEESNOTE("analyzing hash table extent #" << ext << " of " << m_extents);
bool duplicate_bugs_found = false;
auto lock = lock_extent_by_index(ext);
}
});
}
- m_prefetch_running = false;
BEESNOTE("calculating hash table statistics");
unique_lock<mutex>
BeesHashTable::lock_extent_by_hash(HashType hash)
{
- BEESTOOLONG("fetch_missing_extent for hash " << to_hex(hash));
+ BEESTOOLONG("lock_extent_by_hash " << to_hex(hash));
return lock_extent_by_index(hash_to_extent_index(hash));
}
-void
-BeesHashTable::fetch_missing_extent_by_index(uint64_t extent_index)
-{
- BEESNOTE("checking hash extent #" << extent_index << " of " << m_extents << " extents");
- auto lock = lock_extent_by_index(extent_index);
- if (!m_extent_metadata.at(extent_index).m_missing) {
- return;
- }
-
- if (!m_persistent) {
- // Memory-only mode: extent is zero-initialised from MAP_ANONYMOUS.
- m_extent_metadata.at(extent_index).m_missing = false;
- return;
- }
-
- // OK we have to read this extent
- BEESNOTE("fetching hash extent #" << extent_index << " of " << m_extents << " extents");
- BEESTRACE("Fetching hash extent #" << extent_index << " of " << m_extents << " extents");
- BEESTOOLONG("Fetching hash extent #" << extent_index << " of " << m_extents << " extents");
-
- uint8_t *const dirty_extent = m_extent_ptr[extent_index].p_byte;
- uint8_t *const dirty_extent_end = m_extent_ptr[extent_index + 1].p_byte;
- const size_t dirty_extent_size = dirty_extent_end - dirty_extent;
- const size_t dirty_extent_offset = dirty_extent - m_byte_ptr;
-
- // If the read fails don't retry, just go with whatever data we have
- m_extent_metadata.at(extent_index).m_missing = false;
-
- catch_all([&]() {
- BEESTOOLONG("pread(fd " << m_fd << " '" << name_fd(m_fd)<< "', length " << to_hex(dirty_extent_end - dirty_extent) << ", offset " << to_hex(dirty_extent - m_byte_ptr) << ")");
- pread_or_die(m_fd, dirty_extent, dirty_extent_size, dirty_extent_offset);
-
- // Only count extents successfully read
- BEESCOUNT(hash_extent_in);
-
- // Won't need that again
- bees_unreadahead(m_fd, dirty_extent_offset, dirty_extent_size);
-
- // If we are in prefetch, give the kernel a hint about the next extent
- if (m_prefetch_running) {
- // Use the kernel readahead here, because it might work for this use case
- readahead(m_fd, dirty_extent_offset + dirty_extent_size, dirty_extent_size);
- }
- });
-
- Cell *cell = m_extent_ptr[extent_index ].p_buckets[0].p_cells;
- Cell *cell_end = m_extent_ptr[extent_index + 1].p_buckets[0].p_cells;
- size_t toxic_cleared_count = 0;
- set<BeesHashTable::Cell> seen_it(cell, cell_end);
- while (cell < cell_end) {
- if (cell->e_addr & BeesAddress::c_toxic_mask) {
- ++toxic_cleared_count;
- cell->e_addr &= ~BeesAddress::c_toxic_mask;
- // Clearing the toxic bit might mean we now have a duplicate.
- // This could be due to a race between two
- // inserts, one finds the extent toxic while the
- // other does not. That's arguably a bug elsewhere,
- // but we should rewrite the whole extent lookup/insert
- // loop, not spend time fixing code that will be
- // thrown out later anyway.
- // If there is a cell that is identical to this one
- // except for the toxic bit, then we don't need this one.
- if (seen_it.count(*cell)) {
- cell->e_addr = 0;
- cell->e_hash = 0;
- }
- }
- ++cell;
- }
- if (toxic_cleared_count) {
- BEESLOGDEBUG("Cleared " << toxic_cleared_count << " hashes while fetching hash table extent " << extent_index);
- }
-}
-
-void
-BeesHashTable::fetch_missing_extent_by_hash(HashType hash)
-{
- uint64_t extent_index = hash_to_extent_index(hash);
- BEESNOTE("waiting to fetch hash extent #" << extent_index << " of " << m_extents << " extents");
-
- fetch_missing_extent_by_index(extent_index);
-}
-
vector<BeesHashTable::Cell>
BeesHashTable::find_cell(HashType hash)
{
- fetch_missing_extent_by_hash(hash);
BEESTOOLONG("find_cell hash " << BeesHash(hash));
vector<Cell> rv;
auto lock = lock_extent_by_hash(hash);
void
BeesHashTable::erase_hash_addr(HashType hash, AddrType addr)
{
- fetch_missing_extent_by_hash(hash);
BEESTOOLONG("erase hash " << to_hex(hash) << " addr " << addr);
auto lock = lock_extent_by_hash(hash);
auto er = get_cell_range(hash);
bool
BeesHashTable::push_front_hash_addr(HashType hash, AddrType addr)
{
- fetch_missing_extent_by_hash(hash);
BEESTOOLONG("push_front_hash_addr hash " << BeesHash(hash) <<" addr " << BeesAddress(addr));
auto lock = lock_extent_by_hash(hash);
const auto er = get_cell_range(hash);
bool
BeesHashTable::push_random_hash_addr(HashType hash, AddrType addr)
{
- fetch_missing_extent_by_hash(hash);
BEESTOOLONG("push_random_hash_addr hash " << BeesHash(hash) << " addr " << BeesAddress(addr));
auto lock = lock_extent_by_hash(hash);
auto er = get_cell_range(hash);
// mmap + size metadata at the final size.
mmap_and_size_metadata();
+ // Load the whole table into the (anonymous) mapping up front, in the
+ // main thread, before any worker can look up a hash. This replaces
+ // the old per-extent demand-load: random worker faults turned the
+ // read into scattered I/O, so a single sequential pass here is both
+ // faster to steady state and lets the lookup/insert paths assume the
+ // table is always resident. Memory-only mode skips this —
+ // the pages are already zero from MAP_ANONYMOUS.
+ if (m_persistent) {
+ for (uint64_t ext = 0; ext < m_extents; ++ext) {
+ BEESNOTE("loading hash extent #" << ext << " of " << m_extents);
+ uint8_t *const extent_begin = m_extent_ptr[ext ].p_byte;
+ uint8_t *const extent_end = m_extent_ptr[ext + 1].p_byte;
+ pread_or_die(m_fd, extent_begin, extent_end - extent_begin, extent_begin - m_byte_ptr);
+ BEESCOUNT(hash_extent_in);
+ }
+ }
+
// Pre-fault and lock now, while we know which size the host has to
// back. prefetch_loop() used to do this on its first analyze pass,
// but by then the mmap had already been allocated unconditionally