bees-context.o \
bees-fdcache.o \
bees-hash.o \
+ bees-hash-domain.o \
bees-heatmap.o \
bees-lib.o \
bees-log.o \
size-min = 32M+1
size-max = max
+# The [hash.*] section sets defaults inherited by every named [hash.NAME]
+# section. Each named section defines one hash domain.
+#
+# Each concrete [hash.NAME] section must name its own hash function.
+# The function is the defining identity of a hash domain and has no
+# default in [hash.*]. Supported values: crc64, cityhash, xxhash.
+# Other values (btrfs, crc32c, sha256, blake2) are reserved for
+# future implementation.
+#
+# The named domain with the highest priority is used as the default
+# global hash function. Additional domains are validated and stored
+# for forward compatibility with csum-based scanning.
+
+[hash.*]
+
+ # Step between successive hash windows in bytes.
+ # Must equal the btrfs checksum block size (4096).
+ step = 4096
+
+ # Bytes consumed per hash window.
+ # Must equal the btrfs checksum block size (4096).
+ size = 4096
+
+ # Seed / collision-domain IV for explicit domain partitioning.
+ # Must be 0.
+ seed = 0
+
+ # Publish this domain's hashes into beeshash.dat.
+ # Must be yes.
+ insert = yes
+
+ # Lookup priority. Highest value is used first.
+ # 0 disables lookup for this domain.
+ priority = 100
+
+# Legacy hash domain. Inherits all [hash.*] defaults.
+# Produces the same 64-bit table keys as bees v0.11 (backward
+# compatible with existing beeshash.dat files).
+
+[hash.crc64]
+
+ function = crc64
+
)--v1-config--";
static BeesConfig::Register s_reg_config_v1("1", bees_config_v1);
return make_shared<BtrfsIoctlLogicalInoArgs>(0, BEES_MAX_EXTENT_REF_COUNT * extent_ref_size + sizeof(btrfs_data_container));
});
+ // Parse [hash.NAME] sections and wire the primary domain's hash
+ // function into BeesHash. Must happen before any worker threads start
+ // so that BeesHash::set_hash_function is called during single-threaded
+ // startup.
+ m_hash_domains = bees_parse_hash_domains(get_config());
+ const BeesHashDomain &primary = bees_primary_hash_domain(m_hash_domains);
+ {
+ const char *fn_name = [&]() -> const char * {
+ switch (primary.function) {
+ case BeesHashFunction::crc64: return "crc64";
+ case BeesHashFunction::cityhash: return "cityhash";
+ case BeesHashFunction::xxhash: return "xxhash";
+ }
+ return "unknown";
+ }();
+ BEESLOGNOTICE("HASH: primary domain '" << primary.name
+ << "' (function=" << fn_name
+ << ", priority=" << primary.priority << ")");
+ }
+ BeesHash::set_hash_function(primary.function, primary.seed);
+
// Force these to exist now so we don't have recursive locking
// operations trying to access them
fd_cache();
--- /dev/null
+#include "bees-hash-domain.h"
+
+#include "bees-config.h"
+#include "bees.h"
+
+#include "crucible/city.h"
+#include "crucible/crc64.h"
+#include "crucible/xxhash.h"
+
+#include <algorithm>
+#include <stdexcept>
+
+using namespace crucible;
+using namespace std;
+
+uint64_t
+BeesHashDomain::hash(const uint8_t *ptr, size_t len) const
+{
+ uint64_t h = 0;
+ switch (function) {
+ case BeesHashFunction::crc64:
+ h = Digest::CRC::crc64(ptr, len);
+ break;
+ case BeesHashFunction::cityhash:
+ h = CityHash64(reinterpret_cast<const char *>(ptr), len);
+ break;
+ case BeesHashFunction::xxhash:
+ h = XXH64(ptr, len, 0);
+ break;
+ }
+ // seed == 0 (enforced by bees_parse_hash_domains), so
+ // finalization is a no-op. The branch is present so the
+ // dispatch lambda in BeesContext does not need to change when
+ // the seed-zero restriction is lifted.
+ if (seed) {
+ // mix64: a bijective 64-bit finalizer to spread seed bits.
+ // Based on the finalizer from MurmurHash3.
+ uint64_t s = seed;
+ s ^= s >> 33;
+ s *= UINT64_C(0xff51afd7ed558ccd);
+ s ^= s >> 33;
+ s *= UINT64_C(0xc4ceb9fe1a85ec53);
+ s ^= s >> 33;
+ h ^= s;
+ }
+ return h;
+}
+
+static BeesHashFunction
+parse_hash_function(const string &name, const string &val)
+{
+ if (val == "crc64") return BeesHashFunction::crc64;
+ if (val == "cityhash") return BeesHashFunction::cityhash;
+ if (val == "xxhash") return BeesHashFunction::xxhash;
+ // Give a clear message for functions reserved for scan_next_extent.
+ if (val == "btrfs" || val == "crc32c" || val == "sha256" || val == "blake2") {
+ throw runtime_error(
+ "[hash." + name + "] function = '" + val +
+ "' is not supported yet"
+ );
+ }
+ throw runtime_error(
+ "[hash." + name + "] function = " + val +
+ " is unknown; supported values: crc64, cityhash, xxhash"
+ );
+}
+
+vector<BeesHashDomain>
+bees_parse_hash_domains(const BeesConfig &cfg)
+{
+ const set<string> names = cfg.sections("hash");
+
+ if (names.empty()) {
+ throw runtime_error(
+ "No [hash.NAME] sections found in configuration; "
+ "at least one hash domain is required"
+ );
+ }
+
+ vector<BeesHashDomain> domains;
+ domains.reserve(names.size());
+
+ for (const string &name : names) {
+ // Skip the wildcard-defaults pseudo-section.
+ if (name == "*") continue;
+
+ const string prefix = "hash." + name + ".";
+
+ BeesHashDomain d;
+ d.name = name;
+
+ d.function = cfg.get(prefix + "function", [&](const string &v) {
+ return parse_hash_function(name, v);
+ });
+
+ d.step = cfg.get(prefix + "step", [&](const string &v) -> uint64_t {
+ const uint64_t val = bees_parse_size(v);
+ if (val != static_cast<uint64_t>(BLOCK_SIZE_SUMS)) {
+ throw runtime_error(
+ "[hash." + name + "] step = " + v +
+ " is not supported on this branch; step must equal "
+ "btrfs_data_block_size (4096)"
+ );
+ }
+ return val;
+ });
+
+ d.size = cfg.get(prefix + "size", [&](const string &v) -> uint64_t {
+ const uint64_t val = bees_parse_size(v);
+ if (val != static_cast<uint64_t>(BLOCK_SIZE_SUMS)) {
+ throw runtime_error(
+ "[hash." + name + "] size = " + v +
+ " is not supported on this branch; size must equal "
+ "btrfs_data_block_size (4096)"
+ );
+ }
+ return val;
+ });
+
+ d.seed = cfg.get(prefix + "seed", [&](const string &v) -> uint64_t {
+ // Accept decimal integers only.
+ const uint64_t val = stoull(v);
+ if (val != 0) {
+ throw runtime_error(
+ "[hash." + name + "] seed = " + v +
+ " is not supported on this branch; seed must be 0"
+ );
+ }
+ return val;
+ });
+
+ d.insert = cfg.get(prefix + "insert", [&](const string &v) -> bool {
+ const bool val = bees_parse_bool(v);
+ if (!val) {
+ throw runtime_error(
+ "[hash." + name + "] insert = no is not supported yet"
+ );
+ }
+ return val;
+ });
+
+ d.priority = cfg.get(prefix + "priority", [](const string &v) -> int {
+ return static_cast<int>(stoull(v));
+ });
+
+ domains.push_back(std::move(d));
+ }
+
+ // Verify at least one insert domain exists (parser enforced insert = yes
+ // above, so every parsed domain qualifies, but check defensively).
+ const bool any_insert = any_of(domains.begin(), domains.end(),
+ [](const BeesHashDomain &d) { return d.insert; });
+ if (!any_insert) {
+ throw runtime_error(
+ "No insert-enabled [hash.NAME] sections found; "
+ "at least one domain with insert = yes is required"
+ );
+ }
+
+ // Sort descending by priority, stable on config (insertion) order for ties.
+ stable_sort(domains.begin(), domains.end(),
+ [](const BeesHashDomain &a, const BeesHashDomain &b) {
+ return a.priority > b.priority;
+ });
+
+ return domains;
+}
+
+const BeesHashDomain &
+bees_primary_hash_domain(const vector<BeesHashDomain> &domains)
+{
+ // domains is sorted descending by priority; the first insert domain
+ // is therefore the highest-priority one.
+ for (const BeesHashDomain &d : domains) {
+ if (d.insert) return d;
+ }
+ // Should never reach here: bees_parse_hash_domains verifies insert domains exist.
+ throw runtime_error("bees_primary_hash_domain: no insert domain found");
+}
--- /dev/null
+#pragma once
+
+/// @file bees-hash-domain.h
+/// Hash domain configuration: runtime objects for [hash.NAME] config sections.
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+using namespace std;
+
+// Forward declaration to avoid a circular include: bees-config.h includes
+// bees.h, which includes this header.
+class BeesConfig;
+
+/// Hash function algorithm for a configured domain.
+enum class BeesHashFunction {
+ crc64, ///< CRC-64 (current default; backward-compatible with existing beeshash.dat)
+ cityhash, ///< CityHash64
+ xxhash, ///< XXHash64
+};
+
+/// Immutable runtime object for one configured [hash.NAME] section.
+///
+/// On the scan_one_extent path, only the highest-priority insert domain is
+/// used (via BeesHash::set_hash_function). All other domains are validated
+/// and stored for forward compatibility with scan_next_extent.
+struct BeesHashDomain {
+ string name; ///< Section name suffix (e.g. "crc64" for [hash.crc64])
+ BeesHashFunction function; ///< Hash algorithm
+ uint64_t step; ///< Step between windows in bytes (must be 4096 on this branch)
+ uint64_t size; ///< Window size in bytes (must be 4096 on this branch)
+ uint64_t seed; ///< Finalization seed (must be 0 on this branch)
+ bool insert; ///< Publish into beeshash.dat (must be true on this branch)
+ int priority; ///< Lookup priority; 0 disables lookup
+
+ /// Compute the 64-bit hash for a buffer under this domain's function.
+ /// The seed XOR finalization step is included when seed != 0, but seed
+ /// is validated to be 0 on this branch, so the finalization path is
+ /// a no-op here and will activate without interface changes.
+ uint64_t hash(const uint8_t *ptr, size_t len) const;
+};
+
+/// Parse all [hash.NAME] sections from @p cfg.
+///
+/// Validation rules enforced on this branch:
+/// - step must equal 4096 (btrfs_data_block_size)
+/// - size must equal 4096
+/// - seed must be 0
+/// - insert must be true
+/// - function must be one of: crc64, cityhash, xxhash
+/// - no duplicate section names
+/// - at least one insert domain must exist
+///
+/// Throws std::runtime_error on any violation.
+///
+/// Returns domains sorted descending by priority, stable on config-file
+/// order when priorities tie (matching hash-plan.md Phase 1 rule).
+vector<BeesHashDomain> bees_parse_hash_domains(const BeesConfig &cfg);
+
+/// Return a const reference to the domain with the highest priority among
+/// insert-enabled domains.
+///
+/// Precondition: @p domains is non-empty and contains at least one domain
+/// with insert == true (enforced by bees_parse_hash_domains).
+const BeesHashDomain &bees_primary_hash_domain(const vector<BeesHashDomain> &domains);
using namespace crucible;
using namespace std;
-BeesHash::BeesHash(const uint8_t *ptr, size_t len) :
- // m_hash(CityHash64(reinterpret_cast<const char *>(ptr), len))
- m_hash(Digest::CRC::crc64(ptr, len))
+// Process-global hash function selection.
+// Defaults to CRC-64 so BeesHash(ptr, len) works correctly even before
+// BeesContext calls set_hash_function at startup.
+// Set once during single-threaded startup; never changed afterward.
+static BeesHashFunction s_hash_function = BeesHashFunction::crc64;
+static uint64_t s_hash_seed = 0;
+
+void
+BeesHash::set_hash_function(BeesHashFunction fn, uint64_t seed)
+{
+ s_hash_function = fn;
+ s_hash_seed = seed;
+}
+
+BeesHash::BeesHash(const uint8_t *ptr, size_t len)
{
+ // Reuse BeesHashDomain::hash() for the actual computation.
+ // Constructing a temporary here is safe: BeesHashDomain is trivially
+ // copyable and the call is inlined by the optimizer.
+ BeesHashDomain d;
+ d.function = s_hash_function;
+ d.seed = s_hash_seed;
+ m_hash = d.hash(ptr, len);
}
ostream &
#include "bees-fwd.h"
#include "bees-fdcache.h"
+#include "bees-hash-domain.h"
#include "bees-lib.h"
#include "bees-log.h"
#include "bees-reporter.h"
void wait_for_transid(const uint64_t count);
};
-/// Thin wrapper around a CityHash64 content hash value.
+/// Thin wrapper around a content hash value.
/// Provides a named type to avoid confusion with raw uint64_t values.
struct BeesHash {
using Type = uint64_t;
/// Implicit conversion to the raw hash value.
operator Type() const { return m_hash; }
BeesHash& operator=(const Type that) { m_hash = that; return *this; }
- /// Hash @p len bytes starting at @p ptr using CityHash64.
+ /// Hash @p len bytes starting at @p ptr using the configured hash function.
+ /// The function is selected at startup by BeesContext via set_hash_function.
+ /// Before set_hash_function is called the default is CRC-64.
BeesHash(const uint8_t *ptr, size_t len);
+
+ /// Set the process-global hash algorithm and seed.
+ /// Must be called once at startup, before any worker threads are created.
+ /// BeesContext calls this after parsing [hash.NAME] config sections to
+ /// wire the highest-priority domain's algorithm into the constructor above.
+ /// Not thread-safe; safe only during single-threaded startup.
+ static void set_hash_function(BeesHashFunction fn, uint64_t seed);
+
private:
Type m_hash;
Fd m_home_fd; ///< FD to the bees home directory (contains state files).
shared_ptr<BeesConfig> m_config;
+ vector<BeesHashDomain> m_hash_domains; ///< Parsed [hash.NAME] sections, sorted by priority.
shared_ptr<BeesFdCache> m_fd_cache;
shared_ptr<BeesHashTable> m_hash_table;
shared_ptr<BeesRoots> m_roots;