}
}
- // Same limits the DP enforces during the walk (scan_next_rewrite_acceptable
- // remains the sole final authority; this only keeps the fallback honest).
- if (dedupe_ops > policy.m_dedupe_max) return nullopt;
- if (copy_ops > policy.m_copy_max) return nullopt;
+ // Same total-op limit the exact DP enforces during its walk; the per-type
+ // caps are scan_next_rewrite_acceptable's authority, and hole_ops was
+ // already bounded by the caller's pre-check. This only keeps the fallback
+ // honest against the total limit.
if (dedupe_ops + copy_ops + in.m_hole_ops > policy.m_total_max) return nullopt;
result.m_matched_blocks = matched_blocks;
// remaining regions (do-nothing-plan.md, the "16000 regions, all paths
// already over total-max" worst case). scan_next_rewrite_acceptable
// stays the single accept authority; this only prunes.
+ // The search always collapses the (dedupe, copy) op split (see the state
+ // machinery below), so during the walk it enforces only the *total* op
+ // limit — the collapsed dimension. The per-type dedupe/copy/hole caps are
+ // enforced by scan_next_rewrite_acceptable, the sole accept authority, on
+ // the final covering. For the common config where each per-type cap is
+ // >= total-max this is exactly equivalent (dedupe_ops <= total_ops <=
+ // total_max <= dedupe_max, so a per-type check could never bind here); when
+ // an operator sets total-max above a per-type cap, the search optimizes the
+ // total and any covering that exceeds a per-type cap is rejected at
+ // acceptance (do-nothing wins) rather than paid for with the removed exact
+ // 2-D search's ~O(V^4) cost.
const auto within_limits = [&](size_t dedupe_ops, size_t copy_ops) -> bool {
- if (dedupe_ops > policy.m_dedupe_max) return false;
- if (copy_ops > policy.m_copy_max) return false;
- if (dedupe_ops + copy_ops + in.m_hole_ops > policy.m_total_max) {
- return false;
- }
- return true;
+ return dedupe_ops + copy_ops + in.m_hole_ops <= policy.m_total_max;
};
// A DP state at a boundary vertex. copy_presence distinguishes
(bytes + in.m_sums_block_size - 1) / in.m_sums_block_size);
};
- // Whether the per-type dedupe/copy limits can bind before the total
- // limit. When they cannot (the common case — all default to 100, so
- // dedupe_ops/copy_ops are each capped by total_max anyway), forward
- // feasibility depends only on the *total* op count, not on how it
- // splits between dedupe and copy. The covering's debt likewise depends
- // only on the total (refs_added counts F = total + hole; the split does
- // not appear), so in that regime two states with the same total and
- // copy_presence are interchangeable and we collapse the (dedupe, copy)
- // Pareto dimensions to their sum. That bounds the live state set to
- // O(total_max) per vertex instead of letting it grow with the boundary
- // count — without it the search was ~O(V^4) (M=250 took 46 s).
- const bool collapse_ops =
- policy.m_dedupe_max >= policy.m_total_max
- && policy.m_copy_max >= policy.m_total_max;
-
- // total_ops of a state — the collapsed cost/limit dimension.
+ // The search always collapses the (dedupe, copy) Pareto dimensions to
+ // their sum. Forward feasibility depends only on the total op count (the
+ // total limit is the only op limit the walk enforces — see within_limits),
+ // and the covering's debt depends only on the total (refs_added counts
+ // F = total + hole; the split does not appear), so two states with the same
+ // total and copy_presence are interchangeable. Collapsing bounds the live
+ // state set to O(total_max) per vertex instead of letting it grow with the
+ // boundary count V. The exact 2-D (dedupe, copy) tracking that this
+ // replaced was ~O(V^4) (M=250 took 46 s); it was only more accurate when
+ // total-max exceeds a per-type cap, a case not worth its cost now that the
+ // acceptance gate rejects any over-per-type covering and the vertex budget
+ // (rewrite.plan-max-vertices) bounds the search regardless.
const auto total_ops = [](const State &s) -> size_t {
return s.m_dedupe_ops + s.m_copy_ops;
};
- // True when state @a dominates state @b (same copy_presence assumed):
- // no worse on modelled cost and no less feasible for any continuation.
- // Used only on the rare non-collapse path; the collapse path inlines an
- // equivalent total_ops comparison into the staircase below.
- const auto dominates = [&](const State &a, const State &b) -> bool {
- if (a.m_partial_debt > b.m_partial_debt) {
- return false;
- }
- if (collapse_ops) {
- return total_ops(a) <= total_ops(b);
- }
- return a.m_dedupe_ops <= b.m_dedupe_ops
- && a.m_copy_ops <= b.m_copy_ops;
- };
-
// Per-vertex live states, split by copy_presence (pruning is only valid
// within one class). Within a class the hot search keys — total_ops and
// partial_debt — live in dense parallel arrays, so the staircase binary
};
// Insert @p s into its copy_presence class, applying dominated-state
- // pruning. Collapse regime: a Pareto staircase keyed on total_ops
- // (ascending) with strictly-descending debt, so domination is a binary
- // search over the dense key arrays and insertion a memmove. Rare
- // non-collapse regime: a linear 2D-Pareto rebuild over the payload.
+ // pruning via a Pareto staircase keyed on total_ops (ascending) with
+ // strictly-descending debt, so domination is a binary search over the
+ // dense key arrays and insertion a memmove.
const auto add_state = [&](VertexBucket &vb, State s) {
StateClass &C = vb.m_cp[s.m_copy_presence ? 1 : 0];
- if (collapse_ops) {
- const uint64_t t = total_ops(s);
- const double d = s.m_partial_debt;
- // First index with total_ops >= t.
- size_t lo = 0, hi = C.size();
- while (lo < hi) {
- const size_t mid = (lo + hi) / 2;
- if (C.m_total[mid] < t) {
- lo = mid + 1;
- } else {
- hi = mid;
- }
- }
- // Dominated by an incumbent with total_ops <= t and debt <= d?
- // The staircase makes the entry with the largest total_ops <= t
- // the lowest-debt one among them, so two probes suffice.
- if (lo < C.size() && C.m_total[lo] == t
- && C.m_debt[lo] <= d) {
- return;
- }
- if (lo > 0 && C.m_debt[lo - 1] <= d) {
- return;
- }
- // Not dominated: evict the incumbents s dominates (total_ops >= t
- // and debt >= d — a contiguous run from lo, since debt descends),
- // then insert s in their place.
- size_t ev = lo;
- while (ev < C.size() && C.m_debt[ev] >= d) {
- ++ev;
+ const uint64_t t = total_ops(s);
+ const double d = s.m_partial_debt;
+ // First index with total_ops >= t.
+ size_t lo = 0, hi = C.size();
+ while (lo < hi) {
+ const size_t mid = (lo + hi) / 2;
+ if (C.m_total[mid] < t) {
+ lo = mid + 1;
+ } else {
+ hi = mid;
}
- C.erase_range(lo, ev);
- C.insert_at(lo, std::move(s));
+ }
+ // Dominated by an incumbent with total_ops <= t and debt <= d?
+ // The staircase makes the entry with the largest total_ops <= t
+ // the lowest-debt one among them, so two probes suffice.
+ if (lo < C.size() && C.m_total[lo] == t
+ && C.m_debt[lo] <= d) {
return;
}
- for (size_t i = 0; i < C.size(); ++i) {
- if (dominates(C.m_state[i], s)) {
- return; // dominated by an incumbent
- }
+ if (lo > 0 && C.m_debt[lo - 1] <= d) {
+ return;
}
- StateClass keep;
- for (size_t i = 0; i < C.size(); ++i) {
- if (!dominates(s, C.m_state[i])) {
- keep.push_back(std::move(C.m_state[i]));
- }
+ // Not dominated: evict the incumbents s dominates (total_ops >= t
+ // and debt >= d — a contiguous run from lo, since debt descends),
+ // then insert s in their place.
+ size_t ev = lo;
+ while (ev < C.size() && C.m_debt[ev] >= d) {
+ ++ev;
}
- keep.push_back(std::move(s));
- C = std::move(keep);
+ C.erase_range(lo, ev);
+ C.insert_at(lo, std::move(s));
};
// Cheap allocation-free pre-check: would a candidate with these counts
// so the (dominant) majority of tentative edges that prune away cost no
// allocation. Must be a strict subset of what add_state rejects, so it
// can never skip a state add_state would keep: it mirrors add_state's
- // domination test exactly (collapse staircase / non-collapse 2D), and
- // the non-collapse path returns false (never pre-filters).
+ // staircase domination test exactly.
const auto dominated = [&](const VertexBucket &vb, bool cp,
size_t dedupe_ops, size_t copy_ops, double d) -> bool {
const StateClass &C = vb.m_cp[cp ? 1 : 0];
- if (!collapse_ops) {
- for (size_t i = 0; i < C.size(); ++i) {
- const State &e = C.m_state[i];
- if (e.m_partial_debt <= d
- && e.m_dedupe_ops <= dedupe_ops
- && e.m_copy_ops <= copy_ops) {
- return true;
- }
- }
- return false;
- }
const uint64_t t = dedupe_ops + copy_ops;
size_t lo = 0, hi = C.size();
while (lo < hi) {
}
}
- // Running copy-source front (collapse regime only). A copy edge
- // costs a flat +K and one copy op regardless of how many vertices it
- // spans — the cost model charges no per-block copy term (line below
- // adds only K) — so the lowest-debt copy landing at a vertex b is
- // always (best copy source at any a < b) + (1 op, K). There is thus
- // no need to fan a copy edge out from every source vertex to every
- // downstream target: sweeping one Pareto front of sources left to
- // right deposits each landing in O(states-in-front) instead of O(V)
- // per source state. That removes the search's dominant cost on
- // overlap-dense extents, where the boundary count V (production has
- // hit ~3850 in one region) made the old per-state copy fan-out
- // O(V^2 * S) and stalled the planner. The front is keyed like
- // StateClass (ascending total_ops, strictly descending debt) and also
- // carries the begin offset each run would start from, for the
- // back-trace. It is reset per region — a copy run never spans the
- // hole between data regions. The rare non-collapse regime keeps the
- // exact per-state fan-out inline below, where the (dedupe, copy) split
- // is not collapsible and total_ops is not a sufficient key.
+ // Running copy-source front. A copy edge costs a flat +K and one copy
+ // op regardless of how many vertices it spans — the cost model charges
+ // no per-block copy term (line below adds only K) — so the lowest-debt
+ // copy landing at a vertex b is always (best copy source at any a < b)
+ // + (1 op, K). There is thus no need to fan a copy edge out from every
+ // source vertex to every downstream target: sweeping one Pareto front
+ // of sources left to right deposits each landing in O(states-in-front)
+ // instead of O(V) per source state. That removes the search's dominant
+ // cost on overlap-dense extents, where the boundary count V (production
+ // has hit ~3850 in one region) made a per-state copy fan-out O(V^2 * S)
+ // and stalled the planner. The front is keyed like StateClass
+ // (ascending total_ops, strictly descending debt) and also carries the
+ // begin offset each run would start from, for the back-trace. It is
+ // reset per region — a copy run never spans the hole between regions.
vector<uint64_t> cf_total; // staircase key, ascending
vector<double> cf_debt; // strictly descending, index-aligned
vector<State> cf_state; // payload, index-aligned
cf_state.insert(cf_state.begin() + lo, s);
cf_begin.insert(cf_begin.begin() + lo, begin);
};
- const bool copy_front_path = collapse_ops && !in.m_dst_compressed;
+ // A copy edge costs a flat +K and one op regardless of span, so the
+ // lowest-debt copy landing at a vertex is always (best source at any
+ // earlier vertex) + one op — one swept Pareto front of sources serves
+ // every landing, with no per-source fan-out. Only a compressed dst,
+ // which permits no copy edges, disables it.
+ const bool copy_front_path = !in.m_dst_compressed;
for (size_t vi = 0; vi < nverts; ++vi) {
const uint64_t a = verts[vi];
const uint64_t reach = reach_end[vi];
const size_t cover_idx = cover_at[vi];
- // Copy arrivals (collapse fast path). Deposit (front + one copy
- // op) here iff a dedupe can begin at this vertex or it is the
- // region end — the only places a debt-minimizing path usefully
- // stops a copy run. Done before the front absorbs this vertex, so
- // a run can never begin and end at the same boundary.
+ // Copy arrivals. Deposit (front + one copy op) here iff a dedupe
+ // can begin at this vertex or it is the region end — the only
+ // places a debt-minimizing path usefully stops a copy run. Done
+ // before the front absorbs this vertex, so a run can never begin
+ // and end at the same boundary.
if (copy_front_path && (reach > a || vi + 1 == nverts)) {
for (size_t i = 0; i < cf_state.size(); ++i) {
const State &src = cf_state[i];
}
}
- // Copy edges, exact per-state fan-out. Used only in the rare
- // non-collapse regime; the collapse regime is served by the
- // running front above. An unclaimed copy gap is one op no
- // matter how many vertices it spans, and a debt-minimizing
- // path never splits a copy run (two ops, strictly worse) nor
- // ends one anywhere but a dedupe start or the region end — so a
- // copy edge only targets such a boundary.
- if (!copy_front_path && !in.m_dst_compressed
- && within_limits(st.m_dedupe_ops, st.m_copy_ops + 1)) {
- for (size_t vj = vi + 1; vj < nverts; ++vj) {
- const bool dedupe_start = reach_end[vj] > verts[vj];
- if (!dedupe_start && vj != nverts - 1) {
- continue;
- }
- const uint64_t b = verts[vj];
- const double nd = st.m_partial_debt + K;
- if (dominated(at[vj], /*copy_presence=*/true,
- st.m_dedupe_ops, st.m_copy_ops + 1, nd)) {
- continue;
- }
- State ns = st;
- ns.m_copy_ops += 1;
- ns.m_copy_presence = true;
- ns.m_partial_debt = nd;
- ns.m_edges = make_shared<const State::EdgeNode>(
- State::EdgeNode{
- State::Edge{ false, 0, a, b },
- st.m_edges });
- add_state(at[vj], std::move(ns));
- }
- }
+ // Copy edges are handled by the swept copy-source front above
+ // (copy_front_path), not a per-state fan-out from here.
}
}
projects / bees / commitdiff