return;
}
- if (m_crawl_clean == m_crawl_dirty) {
- BEESLOGINFO("Nothing to save");
- return;
- }
-
// Update m_point_ini in-place while holding the lock.
// Extent crawlers: always update, never remove — the tier list is fixed in config
// and deactivated tiers should retain their position for when they are reactivated.
if (m_point_ini.sections("scan").empty()) {
BEESLOGWARN("Crawl state empty!");
- m_crawl_clean = m_crawl_dirty;
return;
}
const string ini_string = m_point_ini.write();
- const auto crawl_saved = m_crawl_dirty;
+
+ // Nothing new to save if the serialised state is byte-identical to the
+ // checkpoint we last wrote or enqueued. This is the whole staleness
+ // test: it cannot drift out of sync with the file the way a separate
+ // dirty counter can, because it compares the bytes that would be
+ // written against the bytes that were.
+ if (ini_string == m_point_saved) {
+ BEESLOGINFO("Nothing to save");
+ return;
+ }
if (!m_checkpoint_defer) {
// Immediate write (current behaviour): write while lock is temporarily released.
- // We must re-acquire the lock before updating m_crawl_clean, and before
+ // We must re-acquire the lock before updating m_point_saved, and before
// returning to the caller (who holds the lock as a postcondition). Use a
// try/catch so a throw from m_point_file.write() still re-locks the mutex
// before the exception propagates -- otherwise save_point() would call
throw;
}
m_mutex.lock();
- m_crawl_clean = crawl_saved;
+ // Only after the write succeeded: a throw above leaves m_point_saved
+ // stale, so the next tick retries instead of silently skipping.
+ m_point_saved = ini_string;
} else {
- // Deferred write: enqueue with current writeback counter.
+ // Deferred write: enqueue with current writeback counter. Record the
+ // state as saved at enqueue time, not at write time -- the queue holds
+ // the write obligation and retries on failure, so re-enqueueing an
+ // identical copy would only add a redundant entry to coalesce away.
const auto wc = m_ctx->hash_table()->writeback_extent_count();
const auto seq = m_next_checkpoint_seq++;
m_pending_checkpoints.push_back({ wc, seq, Timer(), ini_string });
- m_crawl_clean = crawl_saved;
+ m_point_saved = ini_string;
BEESLOGINFO("checkpoint queued #" << seq << " age " << static_cast<uint64_t>(m_pending_checkpoints.back().queued.age()) << "s writeback " << wc);
}
}
os << m_point_ini.write();
}
-void
-BeesRoots::crawl_state_set_dirty()
-{
- unique_lock<mutex> lock(m_mutex);
- ++m_crawl_dirty;
-}
-
void
BeesRoots::crawl_state_erase(const BeesCrawlState &bcs)
{
return;
}
- if (m_root_crawl_map.count(bcs.m_root)) {
- m_root_crawl_map.erase(bcs.m_root);
- ++m_crawl_dirty;
- }
+ m_root_crawl_map.erase(bcs.m_root);
}
uint64_t
BeesRoots::writeback_thread()
{
while (true) {
- BEESNOTE("idle, " << (m_crawl_clean != m_crawl_dirty ? "dirty" : "clean"));
-
catch_all([&]() {
BEESNOTE("saving crawler state");
save_point();
});
+ // Report the deferred-checkpoint backlog for the duration of the
+ // sleep below. A deep backlog is normal, not a fault: entries are
+ // released one full hash table sweep after they are queued, and the
+ // hash table flush is intentionally lazy, so a sweep can take days.
+ // Expect roughly (sweep time / checkpoint interval) entries in the
+ // steady state, with one queued and one written per interval.
+ size_t pending_count;
+ {
+ unique_lock<mutex> point_lock(m_mutex);
+ pending_count = m_pending_checkpoints.size();
+ }
+ BEESNOTE("idle, " << pending_count << " checkpoints pending");
+
unique_lock<mutex> lock(m_stop_mutex);
if (m_stop_requested) {
BEESLOGDEBUG("Stop requested in writeback thread");
auto new_bcp = make_shared<BeesCrawl>(m_ctx, new_bcs);
auto new_pair = make_pair(new_bcs.m_root, new_bcp);
m_root_crawl_map.insert(new_pair);
- ++m_crawl_dirty;
}
const auto found = m_root_crawl_map.find(new_bcs.m_root);
THROW_CHECK0(runtime_error, found != m_root_crawl_map.end());
if (!m_extent_crawl_map.count(name)) {
auto new_bcp = make_shared<BeesCrawl>(m_ctx, new_bcs);
m_extent_crawl_map.insert(make_pair(name, new_bcp));
- ++m_crawl_dirty;
}
const auto found = m_extent_crawl_map.find(name);
THROW_CHECK0(runtime_error, found != m_extent_crawl_map.end());
void
BeesCrawl::set_state(const BeesCrawlState &bcs)
{
+ // Take a hold at bcs and drop it immediately, so the low-water mark is
+ // free to advance to bcs as soon as any older holds retire. Unlike
+ // hold_state(), the caller is not pinning the position across any work.
m_state.hold(bcs);
- m_ctx->roots()->crawl_state_set_dirty();
}
void
CrawlMap m_root_crawl_map; ///< Map from root ID to its BeesCrawl (subvol crawlers only).
map<string, shared_ptr<BeesCrawl>> m_extent_crawl_map; ///< Map from tier name to its BeesCrawl (extent crawlers only).
mutex m_mutex;
- /// Generation counter incremented when crawl state is modified.
- /// Writeback fires when dirty != clean.
- /// @note This pair may be a partially implemented feature or debugging remnant.
- /// Current semantics: writeback thread writes checkpoint when dirty != clean,
- /// then advances clean to match dirty.
- uint64_t m_crawl_dirty = 0;
- /// Generation counter advanced to m_crawl_dirty after each successful checkpoint write.
- uint64_t m_crawl_clean = 0;
+ /// Serialised beespoint.ini content of the most recent checkpoint that was
+ /// written (immediate mode) or enqueued (deferred mode). snapshot_point()
+ /// compares against this to decide whether there is anything new to save.
+ ///
+ /// A generation counter cannot be used here: in extent scan mode the crawl
+ /// position advances through ProgressTracker holders, which retire without
+ /// passing through BeesCrawl::set_state(), so no counter bumped on the
+ /// set_state() path can observe ordinary scan progress. The serialised
+ /// state is the only thing that always reflects what would be written.
+ string m_point_saved;
Timer m_crawl_timer;
BeesThread m_crawl_thread;
BeesThread m_writeback_thread;
/// state. If beescrawl.dat is also absent, both maps are left empty (first run).
/// beescrawl.dat is never modified.
void import_legacy();
- void crawl_state_set_dirty();
void crawl_state_erase(const BeesCrawlState &bcs);
bool all_crawlers_finished();
bool all_crawlers_one_pass();