protected:
shared_ptr<BeesContext> m_ctx;
shared_ptr<BeesRoots> m_roots;
+ mutex m_scan_task_mutex;
+ Task m_scan_task;
bool crawl_batch(const shared_ptr<BeesCrawl>& crawl);
+ virtual void start_scan();
+ virtual void scan() = 0;
public:
virtual ~BeesScanMode() {}
BeesScanMode(const shared_ptr<BeesRoots>& roots, const shared_ptr<BeesContext>& ctx);
- virtual bool do_scan() = 0;
- virtual bool scan() = 0;
using CrawlMap = decltype(BeesRoots::m_root_crawl_map);
virtual void next_transid(const CrawlMap &crawl_map) = 0;
virtual const char *ntoa() const = 0;
{
}
+void
+BeesScanMode::start_scan()
+{
+ unique_lock<mutex> lock(m_scan_task_mutex);
+ if (!m_scan_task) {
+ const auto st = shared_from_this();
+ ostringstream oss;
+ oss << "scan_" << ntoa();
+ m_scan_task = Task(oss.str(), [st] {
+ st->scan();
+ });
+ }
+ m_scan_task.idle();
+}
+
bool
BeesScanMode::crawl_batch(const shared_ptr<BeesCrawl>& crawl)
{
using Map = map<SortKey, CrawlMap::mapped_type>;
mutex m_mutex;
shared_ptr<Map> m_sorted;
+ void scan() override;
public:
using BeesScanMode::BeesScanMode;
~BeesScanModeLockstep() override {}
- bool do_scan() override;
- bool scan() override;
void next_transid(const CrawlMap &crawl_map) override;
const char *ntoa() const override;
};
return "LOCKSTEP";
}
-bool
-BeesScanModeLockstep::do_scan()
-{
- return true;
-}
-
-bool
+void
BeesScanModeLockstep::scan()
{
unique_lock<mutex> lock(m_mutex);
lock.unlock();
if (!hold_sorted) {
BEESLOGINFO("called Lockstep scan without a sorted map");
- return false;
+ return;
}
auto &sorted = *hold_sorted;
while (!sorted.empty()) {
const auto insert_rv = sorted.insert(new_value);
THROW_CHECK0(runtime_error, insert_rv.second);
}
- return true;
+ Task::current_task().idle();
+ return;
}
}
- return false;
}
void
}
unique_lock<mutex> lock(m_mutex);
swap(m_sorted, new_map);
+ lock.unlock();
+ start_scan();
}
/// Scan each subvol in round-robin with no synchronization.
using List = list<CrawlMap::mapped_type>;
mutex m_mutex;
shared_ptr<List> m_subvols;
+ void scan() override;
public:
using BeesScanMode::BeesScanMode;
~BeesScanModeIndependent() override {}
- bool do_scan() override;
- bool scan() override;
void next_transid(const CrawlMap &crawl_map) override;
const char *ntoa() const override;
};
return "INDEPENDENT";
}
-bool
-BeesScanModeIndependent::do_scan()
-{
- return true;
-}
-
-bool
+void
BeesScanModeIndependent::scan()
{
unique_lock<mutex> lock(m_mutex);
lock.unlock();
if (!hold_subvols) {
BEESLOGINFO("called Independent scan without a subvol list");
- return false;
+ return;
}
auto &subvols = *hold_subvols;
while (!subvols.empty()) {
const bool rv = crawl_batch(this_crawl);
if (rv) {
subvols.push_back(this_crawl);
- return true;
+ Task::current_task().idle();
+ return;
}
}
- return false;
}
void
}
unique_lock<mutex> lock(m_mutex);
swap(m_subvols, new_subvols);
+ lock.unlock();
+ start_scan();
}
/// Scan each subvol completely, in numerical order, before moving on to the next.
using Map = map<SortKey, CrawlMap::mapped_type>;
mutex m_mutex;
shared_ptr<Map> m_sorted;
+ void scan() override;
public:
using BeesScanMode::BeesScanMode;
~BeesScanModeSequential() override {}
- bool do_scan() override;
- bool scan() override;
void next_transid(const CrawlMap &crawl_map) override;
const char *ntoa() const override;
};
return "SEQUENTIAL";
}
-bool
-BeesScanModeSequential::do_scan()
-{
- return true;
-}
-
-bool
+void
BeesScanModeSequential::scan()
{
unique_lock<mutex> lock(m_mutex);
lock.unlock();
if (!hold_sorted) {
BEESLOGINFO("called Sequential scan without a sorted map");
- return false;
+ return;
}
auto &sorted = *hold_sorted;
while (!sorted.empty()) {
const auto this_crawl = sorted.begin()->second;
const bool rv = crawl_batch(this_crawl);
if (rv) {
- return true;
+ Task::current_task().idle();
+ return;
} else {
sorted.erase(sorted.begin());
}
}
- return false;
}
void
}
unique_lock<mutex> lock(m_mutex);
swap(m_sorted, new_map);
+ lock.unlock();
+ start_scan();
}
/// Scan the most recently completely scanned subvols first. Keeps recently added data
using Map = map<SortKey, list<CrawlMap::mapped_type>>;
mutex m_mutex;
shared_ptr<Map> m_sorted;
+ void scan() override;
public:
using BeesScanMode::BeesScanMode;
~BeesScanModeRecent() override {}
- bool do_scan() override;
- bool scan() override;
void next_transid(const CrawlMap &crawl_map) override;
const char *ntoa() const override;
};
return "RECENT";
}
-bool
-BeesScanModeRecent::do_scan()
-{
- return true;
-}
-
-bool
+void
BeesScanModeRecent::scan()
{
unique_lock<mutex> lock(m_mutex);
lock.unlock();
if (!hold_sorted) {
BEESLOGINFO("called Recent scan without a sorted map");
- return false;
+ return;
}
auto &sorted = *hold_sorted;
while (!sorted.empty()) {
const bool rv = crawl_batch(this_crawl);
if (rv) {
this_list.push_back(this_crawl);
- return true;
+ Task::current_task().idle();
+ return;
}
}
}
- return false;
}
void
}
unique_lock<mutex> lock(m_mutex);
swap(m_sorted, new_map);
+ start_scan();
}
/// Scan the extent tree and submit each extent's references in a single batch.
friend ostream& operator<<(ostream &os, const BeesScanModeExtent::ExtentRef& todo);
void init_tasks();
- void run_tasks();
+ void scan() override;
void map_next_extent(uint64_t subvol);
bool crawl_one_extent(const ExtentRef &bior);
void create_extent_map(const uint64_t bytenr, const ProgressTracker<BeesCrawlState>::ProgressHolder& m_hold, uint64_t len);
public:
BeesScanModeExtent(const shared_ptr<BeesRoots>& roots, const shared_ptr<BeesContext>& ctx);
~BeesScanModeExtent() override {}
- bool do_scan() override;
- bool scan() override;
void next_transid(const CrawlMap &crawl_map) override;
const char *ntoa() const override;
};
}
void
-BeesScanModeExtent::run_tasks()
+BeesScanModeExtent::scan()
{
if (should_throttle()) return;
unique_lock<mutex> lock(m_mutex);
+ const auto task_map_copy = m_task_map;
+ lock.unlock();
+
// Good to go, start everything running
- for (const auto &i : m_task_map) {
+ for (const auto &i : task_map_copy) {
i.second.idle();
}
}
BEESCOUNT(crawl_done);
}
-bool
-BeesScanModeExtent::do_scan()
-{
- return false;
-}
-
-bool
-BeesScanModeExtent::scan()
-{
- // This is now driven directly from next_transid
- return false;
-}
-
void
BeesScanModeExtent::next_transid(const CrawlMap &crawl_map)
{
}
// Kick off tasks if they aren't already running
- run_tasks();
+ start_scan();
// Swap in the new crawl map with freshly undeferred crawlers
auto crawl_map_copy = crawl_map;
return true;
}
-bool
-BeesRoots::crawl_roots()
-{
- BEESNOTE("Crawling roots");
- BEESTRACE("Crawling roots");
-
- unique_lock<mutex> lock(m_mutex);
- const auto hold_scanner = m_scanner;
- lock.unlock();
-
- THROW_CHECK0(runtime_error, hold_scanner);
-
- BEESNOTE("Scanning roots in " << hold_scanner->ntoa() << " mode");
- BEESTRACE("scanning roots in " << hold_scanner->ntoa() << " mode");
-
- // Clumsy adapter for legacy scan modes
- if (!hold_scanner->do_scan()) {
- return false;
- }
- if (hold_scanner->scan()) {
- return true;
- }
-
- BEESCOUNT(crawl_done);
-
- const auto ran_out_time = m_crawl_timer.lap();
- BEESLOGINFO("crawl_more ran out of data after " << ran_out_time << "s");
-
- // Do not run again
- return false;
-}
-
void
BeesRoots::clear_caches()
{
// Create the Task that does the crawling
const auto shared_this = shared_from_this();
- const auto crawl_task = Task("crawl_more", [shared_this]() {
- BEESTRACE("crawl_more " << shared_this);
- if (shared_this->crawl_roots()) {
- Task::current_task().idle();
- }
- });
- const auto crawl_new = Task("crawl_new", [shared_this, crawl_task]() {
+ const auto crawl_new = Task("crawl_new", [shared_this]() {
BEESTRACE("crawl_new " << shared_this);
catch_all([&]() {
shared_this->insert_new_crawl();
});
- crawl_task.run();
});
// Monitor transid_max and wake up roots when it changes
projects / bees / commitdiff