]> git.hungrycats.org Git - bees/commitdiff
borrower: make failed batch reacquires ordered, side-effect-free, quiet
authorZygo Blaxell <bees@furryterror.org>
Thu, 3 Sep 2026 01:35:11 +0000 (21:35 -0400)
committerZygo Blaxell <bees@furryterror.org>
Sat, 10 Oct 2026 19:00:06 +0000 (15:00 -0400)
At endgame the only remaining work is a handful of highly-shared ref
clusters, and every planner task restart wakes the whole post-exec
waiter herd of the task that finished.  Each woken task re-ran the
batch reacquire, taking memoized locks one by one until one failed,
then releasing them all — knocking over the locks the other woken
tasks were about to succeed on.  Two tasks whose two-lock memos
overlap in opposite orders can ping-pong this way for minutes,
re-waking their full waiter sets on every iteration.  A full-corpus
run logged 4.36M "memo reacquire contention" lines (64% of the log)
in a 13-minute tail during which the dedupe rate collapsed and CPU
burned on retry churn and log formatting.

Three changes to the batch reacquire in the Borrower constructor:

- Sort the memoized set into canonical (pointer) order, so tasks with
  overlapping memos contend on their lowest common Exclusion first,
  forming one queue on one lock instead of a swap cycle.

- Peek the whole set with Exclusion::locked_by_another before
  acquiring anything.  A wakeup that cannot proceed now touches only
  the failing Exclusion (to establish the restart rendezvous) and
  disturbs no other locks.

- Rate-limit the contention log line to 10/s.  The
  borrower_dead_reacquire counter is unconditional as before; the
  pass-2 race path logs under its own limiter as "memo reacquire
  race" so the two causes stay distinguishable.

Assisted-by: Claude-Code:claude-fable-5
src/bees-borrower.cc

index f2c1d1fdc7c7d384312ab1a25722e308779f5ce2..cce3924a4e8e502c02c18b278549e68de12b2c36 100644 (file)
@@ -7,6 +7,9 @@
 #include "bees.h"
 
 #include "crucible/error.h"
+#include "crucible/time.h"
+
+#include <algorithm>
 
 using namespace crucible;
 using namespace std;
@@ -49,18 +52,63 @@ Borrower::Borrower(shared_ptr<PlannerState> ps) :
                // The Task infrastructure retries via the failing Exclusion's
                // rendezvous queue.
                auto memo = m_planner_state->locked_set();
+
+               // Canonical acquisition order: all restarting tasks whose
+               // memos overlap contend on their lowest common Exclusion
+               // first, forming one queue on one lock instead of an
+               // A-wants-B / B-wants-A swap cycle that re-wakes both
+               // tasks' waiter herds on every iteration.
+               sort(memo.begin(), memo.end());
+
+               const auto task = Task::current_task();
                unique_lock<mutex> lock(s_mutex);
+
+               // Pass 1: side-effect-free peek.  A woken task whose memo
+               // is still contended must not acquire (and then release)
+               // any locks on its way to discovering that — the transient
+               // take-and-release perturbs the locks other woken tasks
+               // are about to succeed on, cascading their failures.  Only
+               // the failing Exclusion is touched, to establish the
+               // rendezvous for the restart.
                for (auto &excl : memo) {
                        if (!excl) continue;
-                       auto el = excl->try_lock(Task::current_task());
-                       if (!el) {
-                               // Rendezvous established on the failing Exclusion.
-                               // m_locks destructs on unwind.  m_dead not set —
-                               // this Borrower never became visible and will be
-                               // discarded as the stack unwinds past its ctor.
+                       if (!excl->locked_by_another(task)) continue;
+                       auto el = excl->try_lock(task);
+                       if (el) {
+                               // Freed between peek and try_lock; el releases
+                               // here (we just took it, nobody queued behind
+                               // it) and pass 2 re-acquires it.
+                               continue;
+                       }
+                       // Rendezvous established on the failing Exclusion.
+                       // m_dead not set — this Borrower never became
+                       // visible and will be discarded as the stack
+                       // unwinds past its ctor.  Log rate-limited: at
+                       // endgame this fires millions of times against a
+                       // handful of hot ref clusters, and the log volume
+                       // is itself a cost.  The counter is unconditional.
+                       BEESCOUNT(borrower_dead_reacquire);
+                       static RateLimiter s_reacquire_log_limit(10);
+                       if (s_reacquire_log_limit.is_ready()) {
                                BEESLOGDEBUG("memo reacquire contention ("
                                        << memo.size() << " locks)");
+                       }
+                       throw Dead();
+               }
+
+               // Pass 2: acquire.  A lock can still be taken by another
+               // thread between the peek and this loop; that follows the
+               // same contention path as before.
+               for (auto &excl : memo) {
+                       if (!excl) continue;
+                       auto el = excl->try_lock(task);
+                       if (!el) {
                                BEESCOUNT(borrower_dead_reacquire);
+                               static RateLimiter s_race_log_limit(10);
+                               if (s_race_log_limit.is_ready()) {
+                                       BEESLOGDEBUG("memo reacquire race ("
+                                               << memo.size() << " locks)");
+                               }
                                throw Dead();
                        }
                        m_locks.push_back({excl, std::move(el)});