local-config-filename = /etc/bees/uuid.d/${UUID}.conf
# The [thread] section controls worker count and loadavg throttling.
-# Notes: computed workers = max(thread-min, min(thread-max, ceil(CPU*factor))).
-# If thread-max is blank, only thread-min and factor apply.
+# thread-min, thread-max, and loadavg-target accept size-style expressions
+# (the same grammar as state.hash.size) with the ${NPROC} (logical CPU count)
+# and ${THREAD_FACTOR} (the thread-factor value below) substitutions, e.g.
+# "${NPROC} * 2" or "min(8, ${NPROC})".
[thread]
# Minimum number of threads to run, regardless of loadavg tracking
- # (legacy -G option).
+ # (legacy -G option). Expression; ${NPROC}/${THREAD_FACTOR} allowed.
thread-min = 0
- # Maximum number of threads to run (legacy -c option).
- # Leave blank for no limit (determined by CPU count).
- thread-max =
+ # Maximum number of threads to run. Expression; ${NPROC} and
+ # ${THREAD_FACTOR} allowed. The value means exactly what it evaluates
+ # to; to cap a CPU-relative count, write the min() yourself. The
+ # legacy -c option overrides this with an exact integer count
+ # (oversubscription allowed; thread-factor then has no effect).
+ thread-max = max(1, ceil(${NPROC} * ${THREAD_FACTOR}))
- # Ratio of bees threads to CPU threads (legacy -C option).
- # thread_count = ceil(cpu_threads * factor)
+ # Ratio of bees threads to CPU threads (legacy -C option), exposed to
+ # the thread expressions above as ${THREAD_FACTOR}.
# formats: 1.5 | 150% | 3/2 | 15:10
thread-factor = 1.0
# Target load average (legacy -g option). 0 disables; otherwise,
- # add or remove workers until loadavg == target.
+ # add or remove workers until loadavg == target. Expression;
+ # ${NPROC}/${THREAD_FACTOR} allowed. May be fractional (e.g. 0.5).
loadavg-target = 0
# Maximum number of extent-scan tasks queued in memory at once.
const auto fs_bytes = ranged_cast<uint64_t>(fs_stat.f_blocks) * ranged_cast<uint64_t>(fs_stat.f_frsize);
insert_map_unique(m_subst_map, "FS_BYTES", to_string(fs_bytes));
+ // Add NPROC: number of logical CPUs, for thread-count expressions like
+ // "${NPROC} * ${THREAD_FACTOR}". hardware_concurrency() may legally
+ // return 0 when it cannot detect the count; clamp to at least 1.
+ insert_map_unique(m_subst_map, "NPROC", to_string(max(1u, thread::hardware_concurrency())));
+
+ // THREAD_FACTOR is not a system fact: it derives from the merged
+ // thread.thread-factor config key and is therefore populated later, in
+ // set_argv(), once all config layers are available.
+
for (const auto &i : m_subst_map) {
BEESLOGDEBUG("subst map: '" << i.first << "' = '" << i.second << "'");
}
BEESLOGDEBUG("defaults config " << m_builtin.get_origin() << " [" << cv_key << "]");
if (!m_global.get_origin().empty()) BEESLOGDEBUG("global config " << m_global.get_origin() << " [" << gcf_key << "]");
if (!m_local.get_origin().empty()) BEESLOGDEBUG("local config " << m_local.get_origin() << " [" << lcf_key << "]");
+
+ // Expose the merged thread.thread-factor as the ${THREAD_FACTOR} subst
+ // variable so thread-count expressions (e.g. the default thread-max,
+ // "max(1, ceil(${NPROC} * ${THREAD_FACTOR}))") can reference it. Store
+ // the normalized numeric value, not the raw string: bees_parse_ratio
+ // accepts "3/2" and "15:10" forms that the size-expression grammar cannot
+ // parse. insert_map_unique is the only writer of this key (the ctor
+ // deliberately leaves it unset), so it cannot throw here.
+ insert_map_unique(m_subst_map, "THREAD_FACTOR",
+ to_string(get("thread.thread-factor", bees_parse_ratio)));
+
}
set<string>
/// mul_expr := atom ( ('*' | '/') atom )*
/// atom := NUMBER SUFFIX?
/// | '(' expr ')'
+/// | 'ceil' '(' expr ')'
/// | 'min' '(' expr ',' expr ')'
/// | 'max' '(' expr ',' expr ')'
/// | 'max' -- numeric_limits<uint64_t>::max()
return v;
}
- // Identifier: 'min'/'max' functions or 'max' sentinel
+ // Identifier: 'ceil'/'min'/'max' functions or 'max' sentinel
if (isalpha(static_cast<unsigned char>(input[pos]))) {
const size_t id_start = pos;
while (pos < input.size() && isalpha(static_cast<unsigned char>(input[pos])))
++pos;
const double a = parse_expr();
skip_ws();
+
+ // One-argument function: 'ceil'
+ if (ident == "ceil") {
+ if (pos >= input.size() || input[pos] != ')') {
+ THROW_ERROR(invalid_argument, "missing ')' after 'ceil()' in '" << input << "'");
+ }
+ ++pos;
+ return ceil(a);
+ }
+
+ // Two-argument functions: 'min', 'max'
if (pos >= input.size() || input[pos] != ',') {
THROW_ERROR(invalid_argument, "expected ',' in '" << ident << "()' in '" << input << "'");
}
};
} // anonymous namespace
-uint64_t
-bees_parse_size(const string &s)
+double
+bees_parse_size_double(const string &s)
{
- BEESTRACE("parsing size '" << s << "'");
+ BEESTRACE("parsing size expression '" << s << "'");
SizeExprParser parser(s);
if (parser.at_end()) {
THROW_ERROR(invalid_argument, "unexpected trailing characters in size expression '" << s << "'");
}
+ THROW_CHECK1(invalid_argument, result, result >= 0.0);
+ return result;
+}
+
+uint64_t
+bees_parse_size(const string &s)
+{
+ const double result = bees_parse_size_double(s);
+
if (isinf(result)) {
return numeric_limits<uint64_t>::max();
}
- THROW_CHECK1(invalid_argument, result, result >= 0.0);
return static_cast<uint64_t>(result);
}
/// An optional @c "+N" addend is supported for half-open range boundaries
/// (e.g. @c "512K+1" as the lower bound of the next tier above 512K).
uint64_t bees_parse_size(const string& str);
+/// Evaluate the same size expression as bees_parse_size() but return the
+/// non-negative @c double result without truncating to an integer. Used for
+/// values that are inherently fractional, e.g. a target load average.
+double bees_parse_size_double(const string& str);
/// Parse a non-negative count, or @c "unlimited" for no limit.
uint64_t bees_parse_count(const string& str);
/// Parse a duration string with mandatory time suffix: @c s (seconds), @c m (minutes),
BEESTRACE("Set up worker thread pool");
- const auto thread_factor = m_config->get("thread.thread-factor", [](const string &s) {
- const auto factor = bees_parse_ratio(s);
- THROW_CHECK1(out_of_range, factor, factor >= 0);
- return factor;
- });
- BEESLOGINFO("worker thread pool size factor " << thread_factor << " [thread.thread-factor]");
-
+ // thread-min, thread-max, and loadavg-target are size-expression values
+ // resolved through ${VAR} substitution (notably ${NPROC} and the merged
+ // ${THREAD_FACTOR}). CPU-relative arithmetic — e.g. the default
+ // thread-max = "max(1, ceil(${NPROC} * ${THREAD_FACTOR}))" — therefore
+ // lives in the config expression rather than here. thread-factor reaches
+ // the thread count only via the ${THREAD_FACTOR} substitution.
const auto thread_count = m_config->get("thread.thread-max", [&](const string &s) {
- const auto thread_max = s.empty()
- ? max(size_t(1), static_cast<size_t>(ceil(thread::hardware_concurrency() * thread_factor)))
- : stoul(s);
+ const auto thread_max = bees_parse_size(m_config->subst(s));
THROW_CHECK1(out_of_range, thread_max, thread_max >= 1);
return thread_max;
});
- const auto load_target = m_config->get("thread.loadavg-target", [](const string &s) { return stoul(s); });
- const auto thread_min = m_config->get("thread.thread-min", [](const string &s) { return stoul(s); });
+ // loadavg-target is a target load average, which is inherently fractional
+ // (and 0 disables throttling): evaluate it as a double so e.g. "0.5" is
+ // not truncated to 0.
+ const auto load_target = m_config->get("thread.loadavg-target", [&](const string &s) {
+ return bees_parse_size_double(m_config->subst(s));
+ });
+ const auto thread_min = m_config->get("thread.thread-min", [&](const string &s) {
+ return bees_parse_size(m_config->subst(s));
+ });
if (load_target != 0) {
BEESLOGINFO("setting load average target to " << load_target << " [thread.loadavg-target]");
BEESLOGINFO("setting worker thread pool minimum size to " << thread_min << " [thread.thread-min]");
}
TaskMaster::set_loadavg_target(load_target);
- BEESLOGINFO("setting worker thread pool maximum size to " << thread_count << " [thread.thread-max, thread.thread-factor]");
+ BEESLOGINFO("setting worker thread pool maximum size to " << thread_count << " [thread.thread-max]");
TaskMaster::set_thread_count(thread_count);
BEESTRACE("loading thread.task-queue-max");
-o, --option Set one configuration option in `key=value` format
Load management options:
- -c, --thread-count Worker thread count (default CPU count * factor)
+ -c, --thread-count Exact worker thread count (overrides factor)
[thread.thread-max]
- -C, --thread-factor Worker thread factor (default 1)
+ -C, --thread-factor Worker thread factor: threads per CPU (default 1)
[thread.thread-factor]
-G, --thread-min Minimum worker thread count (default 0)
[thread.thread-min]
}
},
{ .name = "thread-count", .has_arg = required_argument, .val = 'c',
+ // Legacy -c sets an exact thread count, overriding the default
+ // thread-max expression (and thus thread-factor). Values above
+ // the CPU count are honoured (oversubscription), matching the
+ // historical behaviour.
.fn = [&]() { bconfig.set("thread.thread-max", optarg); }
},
{ .name = "loadavg-target", .has_arg = required_argument, .val = 'g',
projects / bees / commitdiff