hash: expose occupancy histogram as a reporter token

Extract the page-occupancy histogram rendering out of prefetch_loop into
BeesHashTable::print_occupancy(), which sums the per-extent fragments and
draws the threshold-doubling graph.  prefetch_loop now calls it where it
used to build the histogram inline, with identical output.

Add a "hashtable" reporter content token that renders the same histogram, so
the occupancy graph can be produced by the reporter rather than only by the
prefetch thread's stats file.  The token reads the hash table through the
context and is skipped when none exists.  Document the token in the config
schema and config-file.md (which also gains the previously undocumented
"idle" token).

Assisted-by: Claude-Code:claude-opus-4-8
Assisted-by: Claude-Code:claude-opus-5
Signed-off-by: Zygo Blaxell <bees@furryterror.org>

diff --git a/docs/config-file.md b/docs/config-file.md
index e33a1998e72cde9a2f5f079fa73e581d562cc937..3c075eb4e1c7de50105d61541a608dd4280adc87 100644 (file)
--- a/docs/config-file.md
+++ b/docs/config-file.md
@@ -400,7 +400,9 @@ not empty.
   * `progress` — extent scan progress table.
   * `workers` — worker thread list.
   * `queue` — pending task queue.
+  * `idle` — idle task queue.
   * `tasks` — task dependency tree.
+  * `hashtable` — hash table page-occupancy histogram.
 
 ### Built-in reports
 

diff --git a/src/bees-config-v1.cc b/src/bees-config-v1.cc
index ae68015027ba50e6fc8dc52423dddf9a5cee7ba7..28e70992afab4d389c16d9531b4a714f3b342806 100644 (file)
--- a/src/bees-config-v1.cc
+++ b/src/bees-config-v1.cc
@@ -176,6 +176,7 @@ static const char bees_config_v1[] = R"--v1-config--(
         #   queue     - pending task queue
         #   idle      - idle task queue
         #   tasks     - task dependency tree
+        #   hashtable - hash table page-occupancy histogram
         content =
 
 # [report.legacy] reproduces the $BEESSTATUS status file.  The default

diff --git a/src/bees-hash.cc b/src/bees-hash.cc
index d3f12bd4a46e308f19a597a46f3748ec0ad52296..6f2a258fe0f83326184b2e7e54417fae0ec3759d 100644 (file)
--- a/src/bees-hash.cc
+++ b/src/bees-hash.cc
@@ -304,64 +304,71 @@ BeesHashTable::writeback_loop()
 }
 
 void
-BeesHashTable::prefetch_loop()
+BeesHashTable::print_occupancy(ostream &os)
 {
-       Uname uname;
-       while (!m_stop_requested) {
-               size_t width = 64;
-               vector<size_t> occupancy(width, 0);
-               size_t occupied_count = 0;
-               const size_t total_count = m_cells;
-
-               // Sum the per-extent occupancy fragments maintained at load and
-               // writeback — no cell walk.  Each extent contributes its cached
-               // bucket-fill histogram.  The old type breakdowns (compressed,
-               // unaligned_eof, toxic) are gone: those were v1 BeesAddress bits
-               // that scan-next neither sets on store nor reads on fetch.
-               for (uint64_t ext = 0; ext < m_extents && !m_stop_requested; ++ext) {
-                       catch_all([&]() {
-                               BEESNOTE("summarizing hash table extent #" << ext << " of " << m_extents);
-                               auto lock = lock_extent_by_index(ext);
-                               const auto &hist = m_extent_metadata.at(ext).m_fill_histogram;
-                               for (size_t fill = 0; fill < hist.size(); ++fill) {
-                                       const size_t buckets_at_fill = hist[fill];
-                                       occupancy.at(fill * width / (1 + c_cells_per_bucket)) += buckets_at_fill;
-                                       occupied_count += fill * buckets_at_fill;
-                               }
-                       });
-               }
-
-               BEESNOTE("calculating hash table statistics");
-
-               vector<string> histogram;
-               vector<size_t> thresholds;
-               size_t threshold = 1;
-               bool threshold_exceeded = false;
-               do {
-                       threshold_exceeded = false;
-                       histogram.push_back(string(width, ' '));
-                       thresholds.push_back(threshold);
-                       for (size_t x = 0; x < width; ++x) {
-                               if (occupancy.at(x) >= threshold) {
-                                       histogram.back().at(x) = '#';
-                                       threshold_exceeded = true;
-                               }
+       const size_t width = 64;
+       vector<size_t> occupancy(width, 0);
+       size_t occupied_count = 0;
+       const size_t total_count = m_cells;
+
+       // Sum the per-extent occupancy fragments maintained at load and
+       // writeback — no cell walk.  Each extent contributes its cached
+       // bucket-fill histogram.  The old type breakdowns (compressed,
+       // unaligned_eof, toxic) are gone: those were v1 BeesAddress bits that
+       // scan-next neither sets on store nor reads on fetch.
+       for (uint64_t ext = 0; ext < m_extents; ++ext) {
+               catch_all([&]() {
+                       BEESNOTE("summarizing hash table extent #" << ext << " of " << m_extents);
+                       auto lock = lock_extent_by_index(ext);
+                       const auto &hist = m_extent_metadata.at(ext).m_fill_histogram;
+                       for (size_t fill = 0; fill < hist.size(); ++fill) {
+                               const size_t buckets_at_fill = hist[fill];
+                               occupancy.at(fill * width / (1 + c_cells_per_bucket)) += buckets_at_fill;
+                               occupied_count += fill * buckets_at_fill;
                        }
-                       threshold *= 2;
-               } while (threshold_exceeded);
-
-               ostringstream out;
-               size_t count = histogram.size();
-               bool first_line = true;
-               for (auto it = histogram.rbegin(); it != histogram.rend(); ++it) {
-                       out << *it << " " << thresholds.at(--count);
-                       if (first_line) {
-                               first_line = false;
-                               out << " pages";
+               });
+       }
+
+       BEESNOTE("calculating hash table statistics");
+
+       vector<string> histogram;
+       vector<size_t> thresholds;
+       size_t threshold = 1;
+       bool threshold_exceeded = false;
+       do {
+               threshold_exceeded = false;
+               histogram.push_back(string(width, ' '));
+               thresholds.push_back(threshold);
+               for (size_t x = 0; x < width; ++x) {
+                       if (occupancy.at(x) >= threshold) {
+                               histogram.back().at(x) = '#';
+                               threshold_exceeded = true;
                        }
-                       out << "\n";
                }
+               threshold *= 2;
+       } while (threshold_exceeded);
+
+       ostringstream out;
+       size_t count = histogram.size();
+       bool first_line = true;
+       for (auto it = histogram.rbegin(); it != histogram.rend(); ++it) {
+               out << *it << " " << thresholds.at(--count);
+               if (first_line) {
+                       first_line = false;
+                       out << " pages";
+               }
+               out << "\n";
+       }
 
+       os << "Hash table page occupancy histogram (" << occupied_count << "/" << total_count << " cells occupied, " << (occupied_count * 100 / total_count) << "%)\n"
+          << out.str() << "0%      |      25%      |      50%      |      75%      |   100% page fill";
+}
+
+void
+BeesHashTable::prefetch_loop()
+{
+       Uname uname;
+       while (!m_stop_requested) {
                ostringstream graph_blob;
 
                graph_blob << "Now:     " << format_time(time(NULL)) << "\n";
@@ -369,10 +376,8 @@ BeesHashTable::prefetch_loop()
                graph_blob << "Version: " << BEES_VERSION << "\n";
                graph_blob << "Kernel:  " << uname.sysname << " " << uname.release << " " << uname.machine << " " << uname.version << "\n";
 
-               graph_blob
-                       << "\nHash table page occupancy histogram (" << occupied_count << "/" << total_count << " cells occupied, " << (occupied_count * 100 / total_count) << "%)\n"
-                       << out.str() << "0%      |      25%      |      50%      |      75%      |   100% page fill";
-
+               graph_blob << "\n";
+               print_occupancy(graph_blob);
                graph_blob << "\n\n";
 
                graph_blob << "TOTAL:\n";

diff --git a/src/bees-reporter.cc b/src/bees-reporter.cc
index 46a0f9dbd084e409d4eee9b719f3cd39e88cb69e..9975de666f1910249143c485466cbecf788aa1c2 100644 (file)
--- a/src/bees-reporter.cc
+++ b/src/bees-reporter.cc
@@ -126,6 +126,14 @@ BeesReporter::write_report(const BeesReportConfig &cfg)
                        TaskMaster::print_idle(oss);
                } else if (token == "tasks") {
                        Task::print_tree(oss);
+               } else if (token == "hashtable") {
+                       // Hash table page-occupancy histogram, summed from the
+                       // per-extent fragments.  Skipped when no hash table exists
+                       // (e.g. before init or in a configuration without one).
+                       if (auto ht = m_ctx->hash_table()) {
+                               ht->print_occupancy(oss);
+                               oss << "\n";
+                       }
                } else {
                        oss << "# token '" << token << "' not implemented in this version of bees\n";
                }

diff --git a/src/bees.h b/src/bees.h
index 4193a6d5baee1510d047d613af15bc28e8f632d3..d713867ceec4f60309d49b271502cc3a0ac0147b 100644 (file)
--- a/src/bees.h
+++ b/src/bees.h
@@ -771,6 +771,8 @@ public:
        void start_threads();
        /// Flush all dirty extents to disk synchronously (no rate limiting).
        void flush();
+       /// Render the page-occupancy histogram (summed from per-extent fragments) to @p os.
+       void print_occupancy(ostream &os);
 
        /// Return all Cells whose hash matches @p hash.
        vector<Cell>    find_cell(HashType hash);