// BEESLOG("BeesResolver br(..., " << bfr << ")");
BEESTRACE("BeesContext::rewrite_file_range calling BeesResolver " << bfr);
BeesResolver br(m_ctx, BeesAddress(bfr.fd(), bfr.begin()));
+ BEESTRACE("BeesContext::rewrite_file_range calling replace_src " << dup_bbd);
// BEESLOG("\treplace_src " << dup_bbd);
br.replace_src(dup_bbd);
BEESCOUNT(scan_rewrite);
}
}
-BeesFileRange
+struct BeesSeenRange {
+ uint64_t bytenr;
+ off_t offset;
+ off_t length;
+};
+
+static
+bool
+operator<(const BeesSeenRange &bsr1, const BeesSeenRange &bsr2)
+{
+ return tie(bsr1.bytenr, bsr1.offset, bsr1.length) < tie(bsr2.bytenr, bsr2.offset, bsr2.length);
+}
+
+static
+__attribute__((unused))
+ostream&
+operator<<(ostream &os, const BeesSeenRange &tup)
+{
+ return os << "BeesSeenRange { " << to_hex(tup.bytenr) << ", " << to_hex(tup.offset) << "+" << pretty(tup.length) << " }";
+}
+
+void
BeesContext::scan_one_extent(const BeesFileRange &bfr, const Extent &e)
{
BEESNOTE("Scanning " << pretty(e.size()) << " "
<< to_hex(e.begin()) << ".." << to_hex(e.end())
<< " " << name_fd(bfr.fd()) );
BEESTRACE("scan extent " << e);
+ BEESTRACE("scan bfr " << bfr);
BEESCOUNT(scan_extent);
// We keep moving this method around
if (e.flags() & Extent::HOLE) {
// Nothing here, dispose of this early
BEESCOUNT(scan_hole);
- return bfr;
+ return;
}
if (e.flags() & Extent::PREALLOC) {
if (m_ctx->dedup(brp)) {
BEESCOUNT(dedup_prealloc_hit);
BEESCOUNTADD(dedup_prealloc_bytes, e.size());
- return bfr;
+ return;
} else {
BEESCOUNT(dedup_prealloc_miss);
}
}
+ // If we already read this extent and inserted it into the hash table, no need to read it again
+ static mutex s_seen_mutex;
+ unique_lock<mutex> lock_seen(s_seen_mutex);
+ const BeesSeenRange tup = {
+ .bytenr = e.bytenr(),
+ .offset = e.offset(),
+ .length = e.size(),
+ };
+ static set<BeesSeenRange> s_seen;
+ if (s_seen.size() > BEES_MAX_EXTENT_REF_COUNT) {
+ s_seen.clear();
+ BEESCOUNT(scan_seen_clear);
+ }
+ const auto seen_rv = s_seen.find(tup) != s_seen.end();
+ if (!seen_rv) {
+ BEESCOUNT(scan_seen_miss);
+ } else {
+ // BEESLOGDEBUG("Skip " << tup << " " << e);
+ BEESCOUNT(scan_seen_hit);
+ return;
+ }
+ lock_seen.unlock();
+
// OK we need to read extent now
bees_readahead(bfr.fd(), bfr.begin(), bfr.size());
map<off_t, pair<BeesHash, BeesAddress>> insert_map;
- set<off_t> noinsert_set;
-
- // Hole handling
- bool extent_compressed = e.flags() & FIEMAP_EXTENT_ENCODED;
- bool extent_contains_zero = false;
- bool extent_contains_nonzero = false;
-
- // Need to replace extent
- bool rewrite_extent = false;
+ set<off_t> dedupe_set;
+ set<off_t> zero_set;
// Pretty graphs
off_t block_count = ((e.size() + BLOCK_MASK_SUMS) & ~BLOCK_MASK_SUMS) / BLOCK_SIZE_SUMS;
BEESTRACE(e << " block_count " << block_count);
string bar(block_count, '#');
- for (off_t next_p = e.begin(); next_p < e.end(); ) {
+ // List of dedupes found
+ list<BeesRangePair> dedupe_list;
+ list<BeesFileRange> copy_list;
+ list<pair<BeesHash, BeesAddress>> front_hash_list;
+ list<uint64_t> invalidate_addr_list;
- // Guarantee forward progress
- off_t p = next_p;
- next_p += BLOCK_SIZE_SUMS;
+ off_t next_p = e.begin();
+ for (off_t p = e.begin(); p < e.end(); p += BLOCK_SIZE_SUMS) {
- off_t bar_p = (p - e.begin()) / BLOCK_SIZE_SUMS;
+ const off_t bar_p = (p - e.begin()) / BLOCK_SIZE_SUMS;
BeesAddress addr(e, p);
// This extent should consist entirely of non-magic blocks
// Calculate the hash first because it lets us shortcut on is_data_zero
BEESNOTE("scan hash " << bbd);
- BeesHash hash = bbd.hash();
+ const BeesHash hash = bbd.hash();
+
+ // Weed out zero blocks
+ BEESNOTE("is_data_zero " << bbd);
+ const bool data_is_zero = bbd.is_data_zero();
+ if (data_is_zero) {
+ bar.at(bar_p) = '0';
+ zero_set.insert(p);
+ BEESCOUNT(scan_zero);
+ continue;
+ }
// Schedule this block for insertion if we decide to keep this extent.
BEESCOUNT(scan_hash_preinsert);
BEESTRACE("Pushing hash " << hash << " addr " << addr << " bbd " << bbd);
insert_map.insert(make_pair(p, make_pair(hash, addr)));
- bar.at(bar_p) = 'R';
+ bar.at(bar_p) = 'i';
- // Weed out zero blocks
- BEESNOTE("is_data_zero " << bbd);
- bool extent_is_zero = bbd.is_data_zero();
- if (extent_is_zero) {
- bar.at(bar_p) = '0';
- if (extent_compressed) {
- if (!extent_contains_zero) {
- // BEESLOG("compressed zero bbd " << bbd << "\n\tin extent " << e);
- }
- extent_contains_zero = true;
- // Do not attempt to lookup hash of zero block
- continue;
- } else {
- BEESLOGINFO("zero bbd " << bbd << "\n\tin extent " << e);
- BEESCOUNT(scan_zero_uncompressed);
- rewrite_extent = true;
- break;
- }
- } else {
- if (extent_contains_zero && !extent_contains_nonzero) {
- // BEESLOG("compressed nonzero bbd " << bbd << "\n\tin extent " << e);
- }
- extent_contains_nonzero = true;
- }
+ // Ensure we fill in the entire insert_map without skipping any non-zero blocks
+ if (p < next_p) continue;
BEESNOTE("lookup hash " << bbd);
- auto found = hash_table->find_cell(hash);
+ const auto found = hash_table->find_cell(hash);
BEESCOUNT(scan_lookup);
set<BeesResolver> resolved_addrs;
// are at least two distinct addresses to look at.
found_addrs.insert(addr);
- for (auto i : found) {
+ for (const auto &i : found) {
BEESTRACE("found (hash, address): " << i);
BEESCOUNT(scan_found);
BeesAddress found_addr(i.e_addr);
-#if 0
// If address already in hash table, move on to next extent.
- // We've already seen this block and may have made additional references to it.
- // The current extent is effectively "pinned" and can't be modified any more.
+ // Only extents that are scanned but not modified are inserted, so if there's
+ // a matching hash:address pair in the hash table:
+ // 1. We have already scanned this extent.
+ // 2. We may have already created references to this extent.
+ // 3. We won't scan this extent again.
+ // The current extent is effectively "pinned" and can't be modified
+ // without rescanning all the existing references.
if (found_addr.get_physical_or_zero() == addr.get_physical_or_zero()) {
+ // No log message because this happens to many thousands of blocks
+ // when bees is interrupted.
+ // BEESLOGDEBUG("Found matching hash " << hash << " at same address " << addr << ", skipping " << bfr);
BEESCOUNT(scan_already);
- return bfr;
+ return;
}
-#endif
// Block must have matching EOF alignment
if (found_addr.is_unaligned_eof() != addr.is_unaligned_eof()) {
// Extents may become non-toxic so give them a chance to expire.
// hash_table->push_front_hash_addr(hash, found_addr);
BEESCOUNT(scan_toxic_hash);
- return bfr;
+ return;
}
// Distinct address, go resolve it
abandon_extent = true;
} else if (!resolved.count()) {
BEESCOUNT(scan_resolve_zero);
- // Didn't find anything, address is dead
- BEESTRACE("matched hash " << hash << " addr " << addr << " count zero");
+ // Didn't find a block at the table address, address is dead
+ BEESLOGDEBUG("Erasing stale addr " << addr << " hash " << hash);
hash_table->erase_hash_addr(hash, found_addr);
} else {
resolved_addrs.insert(resolved);
});
if (abandon_extent) {
- return bfr;
+ return;
}
}
BEESCOUNT(matched_2_or_more);
}
- // No need to do all this unless there are two or more distinct matches
+ // No need to do all this unless there are one or more distinct matches
if (!resolved_addrs.empty()) {
bar.at(bar_p) = 'M';
BEESCOUNT(matched_1_or_more);
BeesFileRange replaced_bfr;
- BeesAddress last_replaced_addr;
for (auto it = resolved_addrs.begin(); it != resolved_addrs.end(); ++it) {
- // FIXME: Need to terminate this loop on replace_dst exception condition
- // catch_all([&]() {
- auto it_copy = *it;
- BEESNOTE("finding one match (out of " << it_copy.count() << ") at " << it_copy.addr() << " for " << bbd);
- BEESTRACE("finding one match (out of " << it_copy.count() << ") at " << it_copy.addr() << " for " << bbd);
- replaced_bfr = it_copy.replace_dst(bbd);
- BEESTRACE("next_p " << to_hex(next_p) << " -> replaced_bfr " << replaced_bfr);
-
- // If we didn't find this hash where the hash table said it would be,
- // correct the hash table.
- if (it_copy.found_hash()) {
- BEESCOUNT(scan_hash_hit);
- } else {
- // BEESLOGDEBUG("erase src hash " << hash << " addr " << it_copy.addr());
- BEESCOUNT(scan_hash_miss);
- hash_table->erase_hash_addr(hash, it_copy.addr());
- }
+ auto it_copy = *it;
+ BEESNOTE("finding one match (out of " << it_copy.count() << ") at " << it_copy.addr() << " for " << bbd);
+ BEESTRACE("finding one match (out of " << it_copy.count() << ") at " << it_copy.addr() << " for " << bbd);
+ // If we replaced extents containing the incoming addr,
+ // push the addr we kept to the front of the hash LRU.
+ auto replaced_brp = it_copy.replace_dst(bbd);
+ replaced_bfr = replaced_brp.second;
+ BEESTRACE("next_p " << to_hex(next_p) << " -> replaced_bfr " << replaced_bfr);
+
+ // If we did find a block, but not this hash, correct the hash table
+ if (it_copy.found_hash()) {
+ BEESCOUNT(scan_hash_hit);
+ } else {
+ BEESLOGDEBUG("Erasing stale hash " << hash << " addr " << it_copy.addr());
+ hash_table->erase_hash_addr(hash, it_copy.addr());
+ BEESCOUNT(scan_hash_miss);
+ }
- if (it_copy.found_dup()) {
- BEESCOUNT(scan_dup_hit);
+ if (it_copy.found_dup()) {
+ THROW_CHECK0(runtime_error, replaced_bfr);
+ BEESCOUNT(scan_dup_hit);
- // FIXME: we will thrash if we let multiple references to identical blocks
- // exist in the hash table. Erase all but the last one.
- if (last_replaced_addr) {
- BEESLOGINFO("Erasing redundant hash " << hash << " addr " << last_replaced_addr);
- hash_table->erase_hash_addr(hash, last_replaced_addr);
- BEESCOUNT(scan_erase_redundant);
- }
- last_replaced_addr = it_copy.addr();
-
- // Invalidate resolve cache so we can count refs correctly
- m_ctx->invalidate_addr(it_copy.addr());
- m_ctx->invalidate_addr(bbd.addr());
-
- // Remove deduped blocks from insert map
- THROW_CHECK0(runtime_error, replaced_bfr);
- for (off_t ip = replaced_bfr.begin(); ip < replaced_bfr.end(); ip += BLOCK_SIZE_SUMS) {
- BEESCOUNT(scan_dup_block);
- noinsert_set.insert(ip);
- if (ip >= e.begin() && ip < e.end()) {
- off_t bar_p = (ip - e.begin()) / BLOCK_SIZE_SUMS;
- bar.at(bar_p) = 'd';
- }
- }
+ // Save this match. If a better match is found later,
+ // it will be replaced. (FIXME: not really...)
+ dedupe_list.push_back(replaced_brp);
- // next_p may be past EOF so check p only
- THROW_CHECK2(runtime_error, p, replaced_bfr, p < replaced_bfr.end());
+ // Push matching block to front of LRU
+ front_hash_list.push_back(make_pair(hash, it_copy.addr()));
- BEESCOUNT(scan_bump);
- next_p = replaced_bfr.end();
- } else {
- BEESCOUNT(scan_dup_miss);
- }
- // });
- }
- if (last_replaced_addr) {
- // If we replaced extents containing the incoming addr,
- // push the addr we kept to the front of the hash LRU.
- hash_table->push_front_hash_addr(hash, last_replaced_addr);
- BEESCOUNT(scan_push_front);
+ off_t bar_p = (p - e.begin()) / BLOCK_SIZE_SUMS;
+ bar.at(bar_p) = '=';
+
+ // Invalidate resolve cache so we can count refs correctly
+ invalidate_addr_list.push_back(it_copy.addr());
+ invalidate_addr_list.push_back(bbd.addr());
+
+ // next_p may be past EOF so check p only
+ THROW_CHECK2(runtime_error, p, replaced_bfr, p < replaced_bfr.end());
+
+ // We may find duplicate ranges of various lengths, so make sure
+ // we don't pick a smaller one
+ next_p = max(next_p, replaced_bfr.end());
+ } else {
+ BEESCOUNT(scan_dup_miss);
+ }
}
} else {
BEESCOUNT(matched_0);
}
}
- // If the extent was compressed and all zeros, nuke entire thing
- if (!rewrite_extent && (extent_contains_zero && !extent_contains_nonzero)) {
- rewrite_extent = true;
- BEESCOUNT(scan_zero_compressed);
+ bool force_insert = false;
+
+ // We don't want to punch holes into compressed extents, unless:
+ // 1. There was dedupe of non-zero blocks, so we always have to copy the rest of the extent
+ // 2. The entire extent is zero and the whole thing can be replaced with a single hole
+ const bool extent_compressed = e.flags() & FIEMAP_EXTENT_ENCODED;
+ if (extent_compressed && dedupe_list.empty() && !insert_map.empty()) {
+ // BEESLOGDEBUG("Compressed extent with non-zero data and no dedupe, skipping");
+ BEESCOUNT(scan_compressed_no_dedup);
+ force_insert = true;
}
- // If we deduped any blocks then we must rewrite the remainder of the extent
- if (!noinsert_set.empty()) {
- rewrite_extent = true;
+ // FIXME: dedupe_list contains a lot of overlapping matches. Get rid of all but one.
+ list<BeesRangePair> dedupe_list_out;
+ dedupe_list.sort([](const BeesRangePair &a, const BeesRangePair &b) {
+ return b.second.size() < a.second.size();
+ });
+ // Shorten each dedupe brp by removing any overlap with earlier (longer) extents in list
+ for (auto i : dedupe_list) {
+ bool insert_i = true;
+ BEESTRACE("i = " << i << " insert_i " << insert_i);
+ for (const auto &j : dedupe_list_out) {
+ BEESTRACE("j = " << j);
+ // No overlap, try next one
+ if (j.second.end() <= i.second.begin() || j.second.begin() >= i.second.end()) {
+ continue;
+ }
+ // j fully overlaps or is the same as i, drop i
+ if (j.second.begin() <= i.second.begin() && j.second.end() >= i.second.end()) {
+ insert_i = false;
+ break;
+ }
+ // i begins outside j, i ends inside j, remove the end of i
+ if (i.second.end() > j.second.begin() && i.second.begin() <= j.second.begin()) {
+ const auto delta = i.second.end() - j.second.begin();
+ if (delta == i.second.size()) {
+ insert_i = false;
+ break;
+ }
+ i.shrink_end(delta);
+ continue;
+ }
+ // i begins inside j, ends outside j, remove the begin of i
+ if (i.second.begin() < j.second.end() && i.second.end() >= j.second.end()) {
+ const auto delta = j.second.end() - i.second.begin();
+ if (delta == i.second.size()) {
+ insert_i = false;
+ break;
+ }
+ i.shrink_begin(delta);
+ continue;
+ }
+ // i fully overlaps j, split i into two parts, push the other part onto dedupe_list
+ if (j.second.begin() > i.second.begin() && j.second.end() < i.second.end()) {
+ auto other_i = i;
+ const auto end_left_delta = i.second.end() - j.second.begin();
+ const auto begin_right_delta = i.second.begin() - j.second.end();
+ i.shrink_end(end_left_delta);
+ other_i.shrink_begin(begin_right_delta);
+ dedupe_list.push_back(other_i);
+ continue;
+ }
+ // None of the sbove. Oops!
+ THROW_CHECK0(runtime_error, false);
+ }
+ if (insert_i) {
+ dedupe_list_out.push_back(i);
+ }
+ }
+ dedupe_list = dedupe_list_out;
+ dedupe_list_out.clear();
+
+ // Count total dedupes
+ uint64_t bytes_deduped = 0;
+ for (const auto &i : dedupe_list) {
+ // Remove deduped blocks from insert map and zero map
+ for (off_t ip = i.second.begin(); ip < i.second.end(); ip += BLOCK_SIZE_SUMS) {
+ BEESCOUNT(scan_dup_block);
+ dedupe_set.insert(ip);
+ zero_set.erase(ip);
+ }
+ bytes_deduped += i.second.size();
}
- // If we need to replace part of the extent, rewrite all instances of it
- if (rewrite_extent) {
- bool blocks_rewritten = false;
+ // Copy all blocks of the extent that were not deduped or zero, but don't copy an entire extent
+ uint64_t bytes_zeroed = 0;
+ if (!force_insert) {
BEESTRACE("Rewriting extent " << e);
off_t last_p = e.begin();
off_t p = last_p;
- off_t next_p;
+ off_t next_p = last_p;
BEESTRACE("next_p " << to_hex(next_p) << " p " << to_hex(p) << " last_p " << to_hex(last_p));
for (next_p = e.begin(); next_p < e.end(); ) {
p = next_p;
- next_p += BLOCK_SIZE_SUMS;
+ next_p = min(next_p + BLOCK_SIZE_SUMS, e.end());
- // BEESLOG("noinsert_set.count(" << to_hex(p) << ") " << noinsert_set.count(p));
- if (noinsert_set.count(p)) {
+ // Can't be both dedupe and zero
+ THROW_CHECK2(runtime_error, zero_set.count(p), dedupe_set.count(p), zero_set.count(p) + dedupe_set.count(p) < 2);
+ if (zero_set.count(p)) {
+ bytes_zeroed += next_p - p;
+ }
+ // BEESLOG("dedupe_set.count(" << to_hex(p) << ") " << dedupe_set.count(p));
+ if (dedupe_set.count(p)) {
if (p - last_p > 0) {
- rewrite_file_range(BeesFileRange(bfr.fd(), last_p, p));
- blocks_rewritten = true;
+ THROW_CHECK2(runtime_error, p, e.end(), p <= e.end());
+ copy_list.push_back(BeesFileRange(bfr.fd(), last_p, p));
}
last_p = next_p;
- } else {
- off_t bar_p = (p - e.begin()) / BLOCK_SIZE_SUMS;
- bar.at(bar_p) = '+';
}
}
BEESTRACE("last");
- if (next_p - last_p > 0) {
- rewrite_file_range(BeesFileRange(bfr.fd(), last_p, next_p));
- blocks_rewritten = true;
+ if (next_p > last_p) {
+ THROW_CHECK2(runtime_error, next_p, e.end(), next_p <= e.end());
+ copy_list.push_back(BeesFileRange(bfr.fd(), last_p, next_p));
}
- if (blocks_rewritten) {
- // Nothing left to insert, all blocks clobbered
- insert_map.clear();
+ }
+
+ // Don't copy an entire extent
+ if (!bytes_zeroed && copy_list.size() == 1 && copy_list.begin()->size() == e.size()) {
+ copy_list.clear();
+ }
+
+ // Count total copies
+ uint64_t bytes_copied = 0;
+ for (const auto &i : copy_list) {
+ bytes_copied += i.size();
+ }
+
+ BEESTRACE("bar: " << bar);
+
+ // Don't do nuisance dedupes part 1: free more blocks than we create
+ THROW_CHECK3(runtime_error, bytes_copied, bytes_zeroed, bytes_deduped, bytes_copied >= bytes_zeroed);
+ const auto cost_copy = bytes_copied - bytes_zeroed;
+ const auto gain_dedupe = bytes_deduped + bytes_zeroed;
+ if (cost_copy > gain_dedupe) {
+ BEESLOGDEBUG("Too many bytes copied (" << pretty(bytes_copied) << ") for bytes deduped (" << pretty(bytes_deduped) << ") and holes punched (" << pretty(bytes_zeroed) << "), skipping extent");
+ BEESCOUNT(scan_skip_bytes);
+ force_insert = true;
+ }
+
+ // Don't do nuisance dedupes part 2: nobody needs more than 100 dedupe/copy ops in one extent
+ if (dedupe_list.size() + copy_list.size() > 100) {
+ BEESLOGDEBUG("Too many dedupe (" << dedupe_list.size() << ") and copy (" << copy_list.size() << ") operations, skipping extent");
+ BEESCOUNT(scan_skip_ops);
+ force_insert = true;
+ }
+
+ // Track whether we rewrote anything
+ bool extent_modified = false;
+
+ // If we didn't delete the dedupe list, do the dedupes now
+ for (const auto &i : dedupe_list) {
+ BEESNOTE("dedup " << i);
+ if (force_insert || m_ctx->dedup(i)) {
+ BEESCOUNT(replacedst_dedup_hit);
+ THROW_CHECK0(runtime_error, i.second);
+ for (off_t ip = i.second.begin(); ip < i.second.end(); ip += BLOCK_SIZE_SUMS) {
+ if (ip >= e.begin() && ip < e.end()) {
+ off_t bar_p = (ip - e.begin()) / BLOCK_SIZE_SUMS;
+ if (bar.at(bar_p) != '=') {
+ if (ip == i.second.begin()) {
+ bar.at(bar_p) = '<';
+ } else if (ip + BLOCK_SIZE_SUMS >= i.second.end()) {
+ bar.at(bar_p) = '>';
+ } else {
+ bar.at(bar_p) = 'd';
+ }
+ }
+ }
+ }
+ extent_modified = !force_insert;
} else {
- // BEESLOG("No blocks rewritten");
- BEESCOUNT(scan_no_rewrite);
+ BEESLOGINFO("dedup failed: " << i);
+ BEESCOUNT(replacedst_dedup_miss);
+ // User data changed while we were looking up the extent, or we have a bug.
+ // We can't fix this, but we can immediately stop wasting effort.
+ return;
}
}
- // We did not rewrite the extent and it contained data, so insert it.
- for (auto i : insert_map) {
- off_t bar_p = (i.first - e.begin()) / BLOCK_SIZE_SUMS;
- BEESTRACE("e " << e << "bar_p = " << bar_p << " i.first-e.begin() " << i.first - e.begin() << " i.second " << i.second.first << ", " << i.second.second);
- if (noinsert_set.count(i.first)) {
- // FIXME: we removed one reference to this copy. Avoid thrashing?
- hash_table->erase_hash_addr(i.second.first, i.second.second);
- // Block was clobbered, do not insert
- // Will look like 'Ddddd' because we skip deduped blocks
- bar.at(bar_p) = 'D';
- BEESCOUNT(inserted_clobbered);
- } else {
+ // Then the copy/rewrites
+ for (const auto &i : copy_list) {
+ if (!force_insert) {
+ rewrite_file_range(i);
+ extent_modified = true;
+ }
+ for (auto p = i.begin(); p < i.end(); p += BLOCK_SIZE_SUMS) {
+ off_t bar_p = (p - e.begin()) / BLOCK_SIZE_SUMS;
+ // Leave zeros as-is because they aren't really copies
+ if (bar.at(bar_p) != '0') {
+ bar.at(bar_p) = '+';
+ }
+ }
+ }
+
+ if (!force_insert) {
+ // Push matched hashes to front
+ for (const auto &i : front_hash_list) {
+ hash_table->push_front_hash_addr(i.first, i.second);
+ BEESCOUNT(scan_push_front);
+ }
+ // Invalidate cached resolves
+ for (const auto &i : invalidate_addr_list) {
+ m_ctx->invalidate_addr(i);
+ }
+ }
+
+ // Don't insert hashes pointing to an extent we just deleted
+ if (!extent_modified) {
+ // We did not rewrite the extent and it contained data, so insert it.
+ // BEESLOGDEBUG("Inserting " << insert_map.size() << " hashes from " << bfr);
+ for (const auto &i : insert_map) {
hash_table->push_random_hash_addr(i.second.first, i.second.second);
- bar.at(bar_p) = '.';
- BEESCOUNT(inserted_block);
+ off_t bar_p = (i.first - e.begin()) / BLOCK_SIZE_SUMS;
+ if (bar.at(bar_p) == 'i') {
+ bar.at(bar_p) = '.';
+ }
+ BEESCOUNT(scan_hash_insert);
}
}
// Visualize
if (bar != string(block_count, '.')) {
- BEESLOGINFO("scan: " << pretty(e.size()) << " " << to_hex(e.begin()) << " [" << bar << "] " << to_hex(e.end()) << ' ' << name_fd(bfr.fd()));
+ BEESLOGINFO(
+ (force_insert ? "skip" : "scan") << ": "
+ << pretty(e.size()) << " "
+ << dedupe_list.size() << "d" << copy_list.size() << "c"
+ << ((bytes_zeroed + BLOCK_SIZE_SUMS - 1) / BLOCK_SIZE_SUMS) << "p {"
+ << to_hex(e.bytenr()) << "+" << to_hex(e.offset()) << "} "
+ << to_hex(e.begin()) << " [" << bar << "] " << to_hex(e.end())
+ << ' ' << name_fd(bfr.fd())
+ );
+ }
+
+ // Put this extent into the recently seen list if we didn't rewrite it,
+ // and remove it if we did.
+ lock_seen.lock();
+ if (extent_modified) {
+ s_seen.erase(tup);
+ BEESCOUNT(scan_seen_erase);
+ } else {
+ // BEESLOGDEBUG("Seen " << tup << " " << e);
+ s_seen.insert(tup);
+ BEESCOUNT(scan_seen_insert);
}
+ lock_seen.unlock();
- // Costs 10% on benchmarks
- // bees_unreadahead(bfr.fd(), bfr.begin(), bfr.size());
- return bfr;
+ // Still hurts benchmarks...or does it?
+ bees_unreadahead(bfr.fd(), bfr.begin(), bfr.size());
}
shared_ptr<Exclusion>
}
Timer one_extent_timer;
scan_one_extent(bfr, e);
+ // BEESLOGDEBUG("Scanned " << e << " " << bfr);
BEESCOUNTADD(scanf_extent_ms, one_extent_timer.age() * 1000);
BEESCOUNT(scanf_extent);
});
projects / bees / commitdiff