* Currently used for observation only; it does not yet change plan
selection.
+* **`plan-max-vertices`**
+ Boundary-vertex budget for the shortest-path coverage search. The exact
+ search is O(V²) in the boundary count V (the distinct match endpoints of a
+ destination extent) and holds that extent's lock for its whole duration, so
+ a single large, heavily-duplicated extent can stall a worker — and every
+ task queued behind that extent — for seconds to minutes. When V exceeds
+ this budget the planner abandons the exact search for a linear-time greedy
+ minimal-interval cover and stops gathering further candidates for that
+ extent. The greedy minimizes fragment count (hence refs added) but does not
+ weigh dedupe-versus-copy tradeoffs, so above the budget plans may be slightly
+ less optimal in exchange for a hard time bound.
+ * Default: `16384`
+ * Accepts a non-negative integer or `unlimited` (which disables the fallback
+ and always runs the exact search).
+
* **`keep-older`**
Enable Plan A: dedupe operations that keep the older copy of
duplicated content. When `yes` (default), the planner builds a
# Default 53 = surveyed btrfs extent-item size.
extent-cost = 53
+ # Boundary-vertex budget for the coverage search. Above this V the
+ # planner uses a linear greedy cover instead of the O(V^2) exact
+ # search. Integer count, or unlimited to disable the fallback.
+ plan-max-vertices = 16384
+
# Maximum hash-conflict restarts per extent. After this many
# restarts the extent is skipped for this scan cycle.
# Integer count. See docs/config-file.md for the rationale.
}
rv.ref_cost = load("rewrite.ref-cost", bees_parse_size);
rv.extent_cost = load("rewrite.extent-cost", bees_parse_size);
+ rv.m_plan_max_vertices = load("rewrite.plan-max-vertices", bees_parse_count);
rv.m_restart_max = load("rewrite.restart-max", bees_parse_count);
rv.m_insert_sample_max = load("rewrite.insert-sample-max", bees_parse_count);
rv.m_insert_conflict_check = load("rewrite.insert-conflict-check", bees_parse_bool);
return c.space_debt(policy);
}
-// Above this many boundary vertices (V), the exact O(V^2) shortest-path DP is
-// abandoned for a linear-time greedy minimal-interval cover. test/bench-bees-
-// plan.cc puts the DP's worst-case knee (collapse on) at V ~ 8192-16384;
-// beyond it a single dense extent can occupy a worker — and, because it holds
-// that dst's Exclusion the whole time, starve every task queued behind it —
-// for seconds to minutes (production has seen an 8-hour single-extent grind
-// with the collapse fast path disabled). The greedy minimizes dedupe-op count
-// (hence fragment count and refs_added / debt) but ignores dedupe-vs-copy debt
-// tradeoffs, so it returns a valid, limit-satisfying, but possibly-suboptimal
-// cover in O(M log M + V).
-//
-// PROTOTYPE: the threshold is a file-scope constant here; productionizing it
-// means a rewrite-policy key (e.g. rewrite.plan-max-vertices) wired across
-// bees.h / bees-config-v2.cc / bees-context.cc / docs/config-file.md per the
-// "update all docs when adding a config option" rule.
-static constexpr size_t BEES_PLAN_V_BUDGET = 8192;
+// The exact O(V^2) shortest-path DP is abandoned above policy.m_plan_max_vertices
+// (rewrite.plan-max-vertices) boundary vertices in favor of a linear-time greedy
+// minimal-interval cover. Rationale: the DP holds the dst extent's Exclusion for
+// its whole duration, so a single dense extent can occupy a worker — and starve
+// every task queued behind it — for seconds to minutes (production has seen an
+// 8-hour single-extent grind with the collapse fast path disabled). The greedy
+// minimizes dedupe-op count (hence fragment count and refs_added / debt) but
+// ignores dedupe-vs-copy debt tradeoffs, so it returns a valid, limit-satisfying,
+// but possibly-suboptimal cover in O(M log M + V).
// Total boundary-vertex count across all data regions — the DP's actual cost
// driver V (region ends plus distinct in-region match endpoints). Cheap
// V-budget: the exact DP below is O(V^2) in the boundary count and holds
// the dst Exclusion for its whole duration, so a high-V extent can wedge a
// worker (and everything queued behind that extent). Above the budget,
- // abandon the exact search for the linear greedy cover.
- if (scan_next_plan_boundary_count(in) > BEES_PLAN_V_BUDGET) {
+ // abandon the exact search for the linear greedy cover. A budget of
+ // UINT64_MAX (rewrite.plan-max-vertices = unlimited) can never be exceeded
+ // and disables the fallback.
+ if (scan_next_plan_boundary_count(in) > policy.m_plan_max_vertices) {
return scan_next_plan_greedy(in, policy);
}
/// extent item (~53 bytes average). Authoritative default lives in
/// bees-config-v2.cc; the C++ initializer is an invalid sentinel.
uint64_t extent_cost = UINT64_MAX;
+ /// Boundary-vertex budget for the shortest-path coverage search.
+ /// The exact search is O(V^2) in the boundary count V and holds the
+ /// dst extent's Exclusion for its whole duration, so a high-V extent
+ /// can stall a worker and every task queued behind that extent. When
+ /// V exceeds this budget the planner abandons the exact search for a
+ /// linear-time greedy cover and stops gathering further candidates.
+ /// See docs/config-file.md; a real default is kept here (not a loud
+ /// sentinel) so directly-constructed policies in tests/benchmarks get
+ /// the exact DP for small inputs. `unlimited` disables the fallback.
+ uint64_t m_plan_max_vertices = 16384;
/// Buffer size for source-side reads in the materialize copy
/// loop and the block_map_fetch hashing pass. Larger values
/// amortise pread syscall and ref-selection cost over more bytes;
assert(secs < 15.0);
}
+static void
+test_search_v_budget_greedy_fallback()
+{
+ // Above policy.m_plan_max_vertices the search abandons the exact O(V^2) DP
+ // for the linear greedy minimal-interval cover. On a fully-coverable
+ // extent (no dedupe-vs-copy tradeoff) the greedy IS the optimum, so forcing
+ // it with a tiny budget must reproduce both the exact DP's result and the
+ // brute-force oracle minimum — same debt, same coverage — proving the
+ // fallback is correct, not merely fast.
+ auto policy = default_policy();
+
+ // Three overlapping matches tile [0,20): minimal cover is 3 dedupes, no
+ // copy. V here is 6 (region ends 0,20 plus interior 6,8,12,14).
+ const auto in = make_inputs(20, { {0, 8}, {6, 14}, {12, 20} }, /*ref=*/1);
+
+ policy.m_plan_max_vertices = 16384; // above V -> exact DP
+ const auto dp = scan_next_plan_shortest_path(in, policy);
+ assert(dp.has_value());
+
+ policy.m_plan_max_vertices = 2; // below V -> greedy fallback
+ const auto greedy = scan_next_plan_shortest_path(in, policy);
+ assert(greedy.has_value());
+
+ // Fully covered by dedupe: all 20 blocks matched, no copy slice, and the
+ // cover is within the op limits.
+ assert(greedy->m_matched_blocks == 20);
+ assert(greedy->m_copy_slices.empty());
+ assert(greedy->m_selected_matches.size() == 3);
+ assert(greedy->m_selected_matches.size() <= policy.m_dedupe_max);
+
+ // Debt reconciles by construction, and equals both the exact DP and the
+ // oracle minimum — greedy is optimal on a coverable extent.
+ assert(greedy->m_debt == result_recomputed_debt(in, policy, *greedy));
+ assert(greedy->m_debt == dp->m_debt);
+ const auto omin = oracle_min_debt(in, policy);
+ assert(omin.has_value());
+ assert(greedy->m_debt == *omin);
+
+ // Copy path: shifting the first match to [1,8) leaves block 0 uncovered.
+ // The greedy must emit exactly one leading copy slice [0,1) and still
+ // reconcile its debt.
+ const auto in_gap = make_inputs(20, { {1, 8}, {6, 14}, {12, 20} }, /*ref=*/1);
+ const auto greedy_gap = scan_next_plan_shortest_path(in_gap, policy);
+ assert(greedy_gap.has_value());
+ assert(greedy_gap->m_copy_slices.size() == 1);
+ assert(greedy_gap->m_copy_slices[0].m_begin == 0);
+ assert(greedy_gap->m_copy_slices[0].m_end == 1 * BLOCK);
+ assert(greedy_gap->m_matched_blocks == 19);
+ assert(greedy_gap->m_debt
+ == result_recomputed_debt(in_gap, policy, *greedy_gap));
+}
+
+static void
+test_search_v_budget_greedy_abandons_over_limit()
+{
+ // When even the greedy minimal cover cannot satisfy the op limits, the
+ // fallback returns nullopt — the intended "abandon this extent" outcome
+ // (it then loses to do-nothing in selection) rather than spending DP time.
+ // This is the documented quality tradeoff: the exact DP could still find an
+ // all-copy plan under the limit here, but a high-V extent is exactly the
+ // case we choose to sacrifice for a hard time bound.
+ auto policy = default_policy();
+ policy.m_dedupe_max = 2;
+ policy.m_total_max = 2;
+ policy.m_plan_max_vertices = 1; // force greedy
+
+ // Five non-overlapping single-block matches tile [0,5): the minimal cover
+ // is 5 dedupe ops, over the limit of 2, so the greedy abandons.
+ const auto in = make_inputs(5,
+ { {0, 1}, {1, 2}, {2, 3}, {3, 4}, {4, 5} }, /*ref=*/1);
+ const auto greedy = scan_next_plan_shortest_path(in, policy);
+ assert(!greedy.has_value());
+}
+
int
main(int, char **)
{
RUN_A_TEST(test_search_brute_force_oracle());
RUN_A_TEST(test_search_compressed_all_or_nothing());
RUN_A_TEST(test_search_large_input_smoke());
+ RUN_A_TEST(test_search_v_budget_greedy_fallback());
+ RUN_A_TEST(test_search_v_budget_greedy_abandons_over_limit());
return 0;
}
projects / bees / commitdiff