dirty_memory_manager: unscramble terminology
Before95f31f37c1("Merge 'dirty_memory_manager: simplify region_group' from Avi Kivity"), we had two region_group objects, one _real_region_group and another _virtual_region_group, each with a set of "soft" and "hard" limits and related functions and members. In95f31f37c1, we merged _real_region_group into _virtual_region_group, but unfortunately the _real_region_group members received the "hard" prefix when they got merged. This overloads the meaning of "hard" - is it related to soft/hard limit or is it related to the real/virtual distinction? This patch applied some renaming to restore consistency. Anything that came from _virtual_region_group now has "virtual" in its name. Anything that came from _real_region_group now has "real" in its name. The terms are still pretty bad but at least they are consistent.
This commit is contained in:
@@ -394,7 +394,7 @@ void set_column_family(http_context& ctx, routes& r) {
|
||||
cf::get_all_cf_all_memtables_off_heap_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
warn(unimplemented::cause::INDEXES);
|
||||
return ctx.db.map_reduce0([](const replica::database& db){
|
||||
return db.dirty_memory_region_group().memory_used();
|
||||
return db.dirty_memory_region_group().virtual_memory_used();
|
||||
}, int64_t(0), std::plus<int64_t>()).then([](int res) {
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
});
|
||||
|
||||
@@ -58,7 +58,7 @@ region_group::add(region* child_r) {
|
||||
assert(!child->_heap_handle);
|
||||
child->_heap_handle = std::make_optional(_regions.push(child));
|
||||
region_group_binomial_group_sanity_check(_regions);
|
||||
update(child_r->occupancy().total_space());
|
||||
update_virtual(child_r->occupancy().total_space());
|
||||
}
|
||||
|
||||
void
|
||||
@@ -67,7 +67,7 @@ region_group::del(region* child_r) {
|
||||
if (child->_heap_handle) {
|
||||
_regions.erase(*std::exchange(child->_heap_handle, std::nullopt));
|
||||
region_group_binomial_group_sanity_check(_regions);
|
||||
update(-child_r->occupancy().total_space());
|
||||
update_virtual(-child_r->occupancy().total_space());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,8 +89,8 @@ region_group::moved(region* old_address, region* new_address) {
|
||||
|
||||
bool
|
||||
region_group::execution_permitted() noexcept {
|
||||
return !(this->under_pressure()
|
||||
|| (_under_hard_pressure));
|
||||
return !(this->under_virtual_pressure()
|
||||
|| (_under_real_pressure));
|
||||
}
|
||||
|
||||
void
|
||||
@@ -134,54 +134,54 @@ region_group::region_group(sstring name,
|
||||
}
|
||||
|
||||
bool region_group::reclaimer_can_block() const {
|
||||
return throttle_threshold() != std::numeric_limits<size_t>::max();
|
||||
return virtual_throttle_threshold() != std::numeric_limits<size_t>::max();
|
||||
}
|
||||
|
||||
void region_group::notify_pressure_relieved() {
|
||||
void region_group::notify_virtual_pressure_relieved() {
|
||||
_relief.signal();
|
||||
}
|
||||
|
||||
bool region_group::do_update_hard_and_check_relief(ssize_t delta) {
|
||||
_hard_total_memory += delta;
|
||||
bool region_group::do_update_real_and_check_relief(ssize_t delta) {
|
||||
_real_total_memory += delta;
|
||||
|
||||
if (_hard_total_memory > hard_throttle_threshold()) {
|
||||
_under_hard_pressure = true;
|
||||
} else if (_under_hard_pressure) {
|
||||
_under_hard_pressure = false;
|
||||
if (_real_total_memory > real_throttle_threshold()) {
|
||||
_under_real_pressure = true;
|
||||
} else if (_under_real_pressure) {
|
||||
_under_real_pressure = false;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void region_group::update_hard(ssize_t delta) {
|
||||
if (do_update_hard_and_check_relief(delta)) {
|
||||
notify_pressure_relieved();
|
||||
void region_group::update_real(ssize_t delta) {
|
||||
if (do_update_real_and_check_relief(delta)) {
|
||||
notify_virtual_pressure_relieved();
|
||||
}
|
||||
}
|
||||
|
||||
void region_group::update(ssize_t delta) {
|
||||
void region_group::update_virtual(ssize_t delta) {
|
||||
// Most-enclosing group which was relieved.
|
||||
bool relief = false;
|
||||
|
||||
_total_memory += delta;
|
||||
_virtual_total_memory += delta;
|
||||
|
||||
if (_total_memory > soft_limit_threshold()) {
|
||||
notify_soft_pressure();
|
||||
if (_virtual_total_memory > virtual_soft_limit_threshold()) {
|
||||
notify_virtual_soft_pressure();
|
||||
} else {
|
||||
notify_soft_relief();
|
||||
notify_virtual_soft_relief();
|
||||
}
|
||||
|
||||
if (_total_memory > throttle_threshold()) {
|
||||
notify_pressure();
|
||||
} else if (under_pressure()) {
|
||||
notify_relief();
|
||||
if (_virtual_total_memory > virtual_throttle_threshold()) {
|
||||
notify_virtual_pressure();
|
||||
} else if (under_virtual_pressure()) {
|
||||
notify_virtual_relief();
|
||||
relief = true;
|
||||
}
|
||||
|
||||
relief |= do_update_hard_and_check_relief(delta);
|
||||
relief |= do_update_real_and_check_relief(delta);
|
||||
|
||||
if (relief) {
|
||||
notify_pressure_relieved();
|
||||
notify_virtual_pressure_relieved();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -65,9 +65,9 @@ using reclaim_start_callback = noncopyable_function<void () noexcept>;
|
||||
using reclaim_stop_callback = noncopyable_function<void () noexcept>;
|
||||
|
||||
struct reclaim_config {
|
||||
size_t hard_limit = std::numeric_limits<size_t>::max();
|
||||
size_t soft_limit = hard_limit;
|
||||
size_t absolute_hard_limit = std::numeric_limits<size_t>::max();
|
||||
size_t virtual_hard_limit = std::numeric_limits<size_t>::max();
|
||||
size_t virtual_soft_limit = virtual_hard_limit;
|
||||
size_t real_hard_limit = std::numeric_limits<size_t>::max();
|
||||
reclaim_start_callback start_reclaiming = [] () noexcept {};
|
||||
reclaim_stop_callback stop_reclaiming = [] () noexcept {};
|
||||
};
|
||||
@@ -150,71 +150,71 @@ public:
|
||||
class region_group : public region_listener {
|
||||
reclaim_config _cfg;
|
||||
|
||||
bool _under_pressure = false;
|
||||
bool _under_soft_pressure = false;
|
||||
bool _under_virtual_pressure = false;
|
||||
bool _under_virtual_soft_pressure = false;
|
||||
|
||||
region_group* _subgroup = nullptr;
|
||||
|
||||
size_t _hard_total_memory = 0;
|
||||
size_t _real_total_memory = 0;
|
||||
|
||||
bool _under_hard_pressure = false;
|
||||
bool _under_real_pressure = false;
|
||||
|
||||
size_t hard_throttle_threshold() const noexcept {
|
||||
return _cfg.absolute_hard_limit;
|
||||
size_t real_throttle_threshold() const noexcept {
|
||||
return _cfg.real_hard_limit;
|
||||
}
|
||||
public:
|
||||
void update_hard(ssize_t delta);
|
||||
void update_real(ssize_t delta);
|
||||
|
||||
size_t hard_memory_used() const noexcept {
|
||||
return _hard_total_memory;
|
||||
size_t real_memory_used() const noexcept {
|
||||
return _real_total_memory;
|
||||
}
|
||||
|
||||
private:
|
||||
bool do_update_hard_and_check_relief(ssize_t delta);
|
||||
bool do_update_real_and_check_relief(ssize_t delta);
|
||||
|
||||
public:
|
||||
bool under_pressure() const noexcept {
|
||||
return _under_pressure;
|
||||
bool under_virtual_pressure() const noexcept {
|
||||
return _under_virtual_pressure;
|
||||
}
|
||||
|
||||
bool over_soft_limit() const noexcept {
|
||||
return _under_soft_pressure;
|
||||
bool over_virtual_soft_limit() const noexcept {
|
||||
return _under_virtual_soft_pressure;
|
||||
}
|
||||
|
||||
void notify_soft_pressure() noexcept {
|
||||
if (!_under_soft_pressure) {
|
||||
_under_soft_pressure = true;
|
||||
void notify_virtual_soft_pressure() noexcept {
|
||||
if (!_under_virtual_soft_pressure) {
|
||||
_under_virtual_soft_pressure = true;
|
||||
_cfg.start_reclaiming();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void notify_soft_relief() noexcept {
|
||||
if (_under_soft_pressure) {
|
||||
_under_soft_pressure = false;
|
||||
void notify_virtual_soft_relief() noexcept {
|
||||
if (_under_virtual_soft_pressure) {
|
||||
_under_virtual_soft_pressure = false;
|
||||
_cfg.stop_reclaiming();
|
||||
}
|
||||
}
|
||||
|
||||
void notify_pressure() noexcept {
|
||||
_under_pressure = true;
|
||||
void notify_virtual_pressure() noexcept {
|
||||
_under_virtual_pressure = true;
|
||||
}
|
||||
|
||||
void notify_relief() noexcept {
|
||||
_under_pressure = false;
|
||||
void notify_virtual_relief() noexcept {
|
||||
_under_virtual_pressure = false;
|
||||
}
|
||||
|
||||
public:
|
||||
size_t throttle_threshold() const noexcept {
|
||||
return _cfg.hard_limit;
|
||||
size_t virtual_throttle_threshold() const noexcept {
|
||||
return _cfg.virtual_hard_limit;
|
||||
}
|
||||
private:
|
||||
size_t soft_limit_threshold() const noexcept {
|
||||
return _cfg.soft_limit;
|
||||
size_t virtual_soft_limit_threshold() const noexcept {
|
||||
return _cfg.virtual_soft_limit;
|
||||
}
|
||||
using region_heap = dirty_memory_manager_logalloc::region_heap;
|
||||
|
||||
size_t _total_memory = 0;
|
||||
size_t _virtual_total_memory = 0;
|
||||
|
||||
region_heap _regions;
|
||||
|
||||
@@ -233,7 +233,7 @@ private:
|
||||
|
||||
bool reclaimer_can_block() const;
|
||||
future<> start_releaser(scheduling_group deferered_work_sg);
|
||||
void notify_pressure_relieved();
|
||||
void notify_virtual_pressure_relieved();
|
||||
friend void region_group_binomial_group_sanity_check(const region_group::region_heap& bh);
|
||||
private: // from region_listener
|
||||
virtual void moved(region* old_address, region* new_address) override;
|
||||
@@ -260,10 +260,10 @@ public:
|
||||
}
|
||||
region_group& operator=(const region_group&) = delete;
|
||||
region_group& operator=(region_group&&) = delete;
|
||||
size_t memory_used() const noexcept {
|
||||
return _total_memory;
|
||||
size_t virtual_memory_used() const noexcept {
|
||||
return _virtual_total_memory;
|
||||
}
|
||||
void update(ssize_t delta);
|
||||
void update_virtual(ssize_t delta);
|
||||
|
||||
// It would be easier to call update, but it is unfortunately broken in boost versions up to at
|
||||
// least 1.59.
|
||||
@@ -277,7 +277,7 @@ public:
|
||||
// the full update cycle even then.
|
||||
virtual void increase_usage(region* r, ssize_t delta) override { // From region_listener
|
||||
_regions.increase(*static_cast<size_tracked_region*>(r)->_heap_handle);
|
||||
update(delta);
|
||||
update_virtual(delta);
|
||||
}
|
||||
|
||||
virtual void decrease_evictable_usage(region* r) override { // From region_listener
|
||||
@@ -286,7 +286,7 @@ public:
|
||||
|
||||
virtual void decrease_usage(region* r, ssize_t delta) override { // From region_listener
|
||||
decrease_evictable_usage(r);
|
||||
update(delta);
|
||||
update_virtual(delta);
|
||||
}
|
||||
|
||||
//
|
||||
@@ -418,7 +418,7 @@ class dirty_memory_manager {
|
||||
void start_reclaiming() noexcept;
|
||||
|
||||
bool has_pressure() const noexcept {
|
||||
return _virtual_region_group.over_soft_limit();
|
||||
return _virtual_region_group.over_virtual_soft_limit();
|
||||
}
|
||||
|
||||
unsigned _extraneous_flushes = 0;
|
||||
@@ -484,39 +484,39 @@ public:
|
||||
}
|
||||
|
||||
void revert_potentially_cleaned_up_memory(logalloc::region* from, int64_t delta) {
|
||||
_virtual_region_group.update_hard(-delta);
|
||||
_virtual_region_group.update(delta);
|
||||
_virtual_region_group.update_real(-delta);
|
||||
_virtual_region_group.update_virtual(delta);
|
||||
_dirty_bytes_released_pre_accounted -= delta;
|
||||
}
|
||||
|
||||
void account_potentially_cleaned_up_memory(logalloc::region* from, int64_t delta) {
|
||||
_virtual_region_group.update_hard(delta);
|
||||
_virtual_region_group.update(-delta);
|
||||
_virtual_region_group.update_real(delta);
|
||||
_virtual_region_group.update_virtual(-delta);
|
||||
_dirty_bytes_released_pre_accounted += delta;
|
||||
}
|
||||
|
||||
void pin_real_dirty_memory(int64_t delta) {
|
||||
_virtual_region_group.update_hard(delta);
|
||||
_virtual_region_group.update_real(delta);
|
||||
}
|
||||
|
||||
void unpin_real_dirty_memory(int64_t delta) {
|
||||
_virtual_region_group.update_hard(-delta);
|
||||
_virtual_region_group.update_real(-delta);
|
||||
}
|
||||
|
||||
size_t real_dirty_memory() const noexcept {
|
||||
return _virtual_region_group.hard_memory_used();
|
||||
return _virtual_region_group.real_memory_used();
|
||||
}
|
||||
|
||||
size_t virtual_dirty_memory() const noexcept {
|
||||
return _virtual_region_group.memory_used();
|
||||
return _virtual_region_group.virtual_memory_used();
|
||||
}
|
||||
|
||||
void notify_soft_pressure() {
|
||||
_virtual_region_group.notify_soft_pressure();
|
||||
_virtual_region_group.notify_virtual_soft_pressure();
|
||||
}
|
||||
|
||||
size_t throttle_threshold() const {
|
||||
return _virtual_region_group.throttle_threshold();
|
||||
return _virtual_region_group.virtual_throttle_threshold();
|
||||
}
|
||||
|
||||
future<> flush_one(replica::memtable_list& cf, flush_permit&& permit) noexcept;
|
||||
@@ -564,9 +564,9 @@ futurize_t<std::result_of_t<Func()>>
|
||||
region_group::run_when_memory_available(Func&& func, db::timeout_clock::time_point timeout) {
|
||||
auto rg = this;
|
||||
bool blocked =
|
||||
!(rg->_blocked_requests.empty() && !rg->under_pressure());
|
||||
!(rg->_blocked_requests.empty() && !rg->under_virtual_pressure());
|
||||
if (!blocked) {
|
||||
blocked = _under_hard_pressure;
|
||||
blocked = _under_real_pressure;
|
||||
}
|
||||
|
||||
if (!blocked) {
|
||||
|
||||
@@ -450,9 +450,9 @@ void backlog_controller::update_controller(float shares) {
|
||||
dirty_memory_manager::dirty_memory_manager(replica::database& db, size_t threshold, double soft_limit, scheduling_group deferred_work_sg)
|
||||
: _db(&db)
|
||||
, _virtual_region_group("memtable (virtual)", dirty_memory_manager_logalloc::reclaim_config{
|
||||
.hard_limit = threshold / 2,
|
||||
.soft_limit = threshold * soft_limit / 2,
|
||||
.absolute_hard_limit = threshold,
|
||||
.virtual_hard_limit = threshold / 2,
|
||||
.virtual_soft_limit = threshold * soft_limit / 2,
|
||||
.real_hard_limit = threshold,
|
||||
.start_reclaiming = std::bind_front(&dirty_memory_manager::start_reclaiming, this)
|
||||
}, deferred_work_sg)
|
||||
, _flush_serializer(1)
|
||||
|
||||
@@ -1731,10 +1731,10 @@ class dirty_mem_mgr():
|
||||
self.ref = ref
|
||||
|
||||
def real_dirty(self):
|
||||
return int(self.ref['_virtual_region_group']['_hard_total_memory'])
|
||||
return int(self.ref['_virtual_region_group']['_real_total_memory'])
|
||||
|
||||
def virt_dirty(self):
|
||||
return int(self.ref['_virtual_region_group']['_total_memory'])
|
||||
return int(self.ref['_virtual_region_group']['_virtual_total_memory'])
|
||||
|
||||
|
||||
def find_instances(type_name):
|
||||
|
||||
@@ -72,8 +72,8 @@ SEASTAR_TEST_CASE(test_region_groups) {
|
||||
});
|
||||
BOOST_REQUIRE_GE(ssize_t(one->occupancy().used_space()), ssize_t(one_count * sizeof(int)));
|
||||
BOOST_REQUIRE_GE(ssize_t(one->occupancy().total_space()), ssize_t(one->occupancy().used_space()));
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.memory_used(), one->occupancy().total_space());
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.hard_memory_used(), one->occupancy().total_space());
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.virtual_memory_used(), one->occupancy().total_space());
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.real_memory_used(), one->occupancy().total_space());
|
||||
|
||||
constexpr size_t two_count = 8 * base_count;
|
||||
std::vector<managed_ref<int>> two_objs;
|
||||
@@ -84,8 +84,8 @@ SEASTAR_TEST_CASE(test_region_groups) {
|
||||
});
|
||||
BOOST_REQUIRE_GE(ssize_t(two->occupancy().used_space()), ssize_t(two_count * sizeof(int)));
|
||||
BOOST_REQUIRE_GE(ssize_t(two->occupancy().total_space()), ssize_t(two->occupancy().used_space()));
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.memory_used(), one->occupancy().total_space() + two->occupancy().total_space());
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.hard_memory_used(), one_and_two.memory_used());
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.virtual_memory_used(), one->occupancy().total_space() + two->occupancy().total_space());
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.real_memory_used(), one_and_two.virtual_memory_used());
|
||||
|
||||
constexpr size_t three_count = 32 * base_count;
|
||||
std::vector<managed_ref<int>> three_objs;
|
||||
@@ -96,7 +96,7 @@ SEASTAR_TEST_CASE(test_region_groups) {
|
||||
});
|
||||
BOOST_REQUIRE_GE(ssize_t(three->occupancy().used_space()), ssize_t(three_count * sizeof(int)));
|
||||
BOOST_REQUIRE_GE(ssize_t(three->occupancy().total_space()), ssize_t(three->occupancy().used_space()));
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.hard_memory_used(), one_and_two.memory_used());
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.real_memory_used(), one_and_two.virtual_memory_used());
|
||||
|
||||
constexpr size_t four_count = 4 * base_count;
|
||||
std::vector<managed_ref<int>> four_objs;
|
||||
@@ -107,7 +107,7 @@ SEASTAR_TEST_CASE(test_region_groups) {
|
||||
});
|
||||
BOOST_REQUIRE_GE(ssize_t(four->occupancy().used_space()), ssize_t(four_count * sizeof(int)));
|
||||
BOOST_REQUIRE_GE(ssize_t(four->occupancy().total_space()), ssize_t(four->occupancy().used_space()));
|
||||
BOOST_REQUIRE_EQUAL(just_four.memory_used(), four->occupancy().total_space());
|
||||
BOOST_REQUIRE_EQUAL(just_four.virtual_memory_used(), four->occupancy().total_space());
|
||||
|
||||
with_allocator(five->allocator(), [] {
|
||||
constexpr size_t five_count = base_count;
|
||||
@@ -120,27 +120,27 @@ SEASTAR_TEST_CASE(test_region_groups) {
|
||||
three->merge(*four);
|
||||
BOOST_REQUIRE_GE(ssize_t(three->occupancy().used_space()), ssize_t((three_count + four_count)* sizeof(int)));
|
||||
BOOST_REQUIRE_GE(ssize_t(three->occupancy().total_space()), ssize_t(three->occupancy().used_space()));
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.hard_memory_used(), one_and_two.memory_used());
|
||||
BOOST_REQUIRE_EQUAL(just_four.memory_used(), 0);
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.real_memory_used(), one_and_two.virtual_memory_used());
|
||||
BOOST_REQUIRE_EQUAL(just_four.virtual_memory_used(), 0);
|
||||
|
||||
three->merge(*five);
|
||||
BOOST_REQUIRE_GE(ssize_t(three->occupancy().used_space()), ssize_t((three_count + four_count)* sizeof(int)));
|
||||
BOOST_REQUIRE_GE(ssize_t(three->occupancy().total_space()), ssize_t(three->occupancy().used_space()));
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.hard_memory_used(), one_and_two.memory_used());
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.real_memory_used(), one_and_two.virtual_memory_used());
|
||||
|
||||
with_allocator(two->allocator(), [&] {
|
||||
two_objs.clear();
|
||||
});
|
||||
two.reset();
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.memory_used(), one->occupancy().total_space());
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.hard_memory_used(), one_and_two.memory_used());
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.virtual_memory_used(), one->occupancy().total_space());
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.real_memory_used(), one_and_two.virtual_memory_used());
|
||||
|
||||
with_allocator(one->allocator(), [&] {
|
||||
one_objs.clear();
|
||||
});
|
||||
one.reset();
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.memory_used(), 0);
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.hard_memory_used(), 0);
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.virtual_memory_used(), 0);
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.real_memory_used(), 0);
|
||||
|
||||
with_allocator(three->allocator(), [&] {
|
||||
three_objs.clear();
|
||||
@@ -149,7 +149,7 @@ SEASTAR_TEST_CASE(test_region_groups) {
|
||||
three.reset();
|
||||
four.reset();
|
||||
five.reset();
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.hard_memory_used(), 0);
|
||||
BOOST_REQUIRE_EQUAL(one_and_two.real_memory_used(), 0);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -209,7 +209,7 @@ private:
|
||||
SEASTAR_TEST_CASE(test_region_groups_basic_throttling) {
|
||||
return seastar::async([] {
|
||||
// singleton hierarchy, only one segment allowed
|
||||
test_region_group simple({ .hard_limit = logalloc::segment_size });
|
||||
test_region_group simple({ .virtual_hard_limit = logalloc::segment_size });
|
||||
auto simple_region = std::make_unique<test_region>();
|
||||
simple_region->listen(&simple);
|
||||
|
||||
@@ -221,11 +221,11 @@ SEASTAR_TEST_CASE(test_region_groups_basic_throttling) {
|
||||
// the group and we'll be okay to do that a second time.
|
||||
auto fut = simple.run_when_memory_available([&simple_region] { simple_region->alloc_small(); }, db::no_timeout);
|
||||
BOOST_REQUIRE_EQUAL(fut.available(), true);
|
||||
BOOST_REQUIRE_EQUAL(simple.memory_used(), logalloc::segment_size);
|
||||
BOOST_REQUIRE_EQUAL(simple.virtual_memory_used(), logalloc::segment_size);
|
||||
|
||||
fut = simple.run_when_memory_available([&simple_region] { simple_region->alloc_small(); }, db::no_timeout);
|
||||
BOOST_REQUIRE_EQUAL(fut.available(), true);
|
||||
BOOST_REQUIRE_EQUAL(simple.memory_used(), logalloc::segment_size);
|
||||
BOOST_REQUIRE_EQUAL(simple.virtual_memory_used(), logalloc::segment_size);
|
||||
|
||||
auto big_region = std::make_unique<test_region>();
|
||||
big_region->listen(&simple);
|
||||
@@ -236,10 +236,10 @@ SEASTAR_TEST_CASE(test_region_groups_basic_throttling) {
|
||||
testlog.info("now = {}", lowres_clock::now().time_since_epoch().count());
|
||||
fut = simple.run_when_memory_available([&simple_region] { simple_region->alloc_small(); }, db::no_timeout);
|
||||
BOOST_REQUIRE_EQUAL(fut.available(), false);
|
||||
BOOST_REQUIRE_GT(simple.memory_used(), logalloc::segment_size);
|
||||
BOOST_REQUIRE_GT(simple.virtual_memory_used(), logalloc::segment_size);
|
||||
|
||||
testlog.info("now = {}", lowres_clock::now().time_since_epoch().count());
|
||||
testlog.info("used = {}", simple.memory_used());
|
||||
testlog.info("used = {}", simple.virtual_memory_used());
|
||||
|
||||
testlog.info("Resetting");
|
||||
|
||||
@@ -248,14 +248,14 @@ SEASTAR_TEST_CASE(test_region_groups_basic_throttling) {
|
||||
// that's up to the internal policies. So to make sure we need to remove the whole region.
|
||||
big_region.reset();
|
||||
|
||||
testlog.info("used = {}", simple.memory_used());
|
||||
testlog.info("used = {}", simple.virtual_memory_used());
|
||||
testlog.info("now = {}", lowres_clock::now().time_since_epoch().count());
|
||||
try {
|
||||
quiesce(std::move(fut));
|
||||
} catch (...) {
|
||||
testlog.info("Aborting: {}", std::current_exception());
|
||||
testlog.info("now = {}", lowres_clock::now().time_since_epoch().count());
|
||||
testlog.info("used = {}", simple.memory_used());
|
||||
testlog.info("used = {}", simple.virtual_memory_used());
|
||||
abort();
|
||||
}
|
||||
testlog.info("now = {}", lowres_clock::now().time_since_epoch().count());
|
||||
@@ -265,14 +265,14 @@ SEASTAR_TEST_CASE(test_region_groups_basic_throttling) {
|
||||
SEASTAR_TEST_CASE(test_region_groups_fifo_order) {
|
||||
// tests that requests that are queued for later execution execute in FIFO order
|
||||
return seastar::async([] {
|
||||
test_region_group rg({.hard_limit = logalloc::segment_size});
|
||||
test_region_group rg({.virtual_hard_limit = logalloc::segment_size});
|
||||
|
||||
auto region = std::make_unique<test_region>();
|
||||
region->listen(&rg);
|
||||
|
||||
// fill the parent. Try allocating at child level. Should not be allowed.
|
||||
region->alloc();
|
||||
BOOST_REQUIRE_GE(rg.memory_used(), logalloc::segment_size);
|
||||
BOOST_REQUIRE_GE(rg.virtual_memory_used(), logalloc::segment_size);
|
||||
|
||||
auto exec_cnt = make_lw_shared<int>(0);
|
||||
std::vector<future<>> executions;
|
||||
@@ -346,7 +346,7 @@ public:
|
||||
(void)with_gate(_reclaimers_done, [this] {
|
||||
return _unleash_reclaimer.get_shared_future().then([this] {
|
||||
_unleashed.set_value();
|
||||
while (_rg.under_pressure()) {
|
||||
while (_rg.under_virtual_pressure()) {
|
||||
size_t reclaimed = test_async_reclaim_region::from_region(_rg.get_largest_region()).evict();
|
||||
_result_accumulator->_reclaim_sizes.push_back(reclaimed);
|
||||
}
|
||||
@@ -370,7 +370,7 @@ public:
|
||||
test_reclaimer(size_t threshold)
|
||||
: _result_accumulator(this)
|
||||
, _rg("test_reclaimer RG", {
|
||||
.hard_limit = threshold,
|
||||
.virtual_hard_limit = threshold,
|
||||
.start_reclaiming = std::bind_front(&test_reclaimer::start_reclaiming, this),
|
||||
}) {}
|
||||
|
||||
@@ -439,7 +439,7 @@ SEASTAR_TEST_CASE(test_no_crash_when_a_lot_of_requests_released_which_change_reg
|
||||
|
||||
auto free_space = memory::stats().free_memory();
|
||||
size_t threshold = size_t(0.75 * free_space);
|
||||
region_group gr(test_name, {.hard_limit = threshold, .soft_limit = threshold});
|
||||
region_group gr(test_name, {.virtual_hard_limit = threshold, .virtual_soft_limit = threshold});
|
||||
auto close_gr = defer([&gr] () noexcept { gr.shutdown().get(); });
|
||||
size_tracked_region r;
|
||||
r.listen(&gr);
|
||||
@@ -458,7 +458,7 @@ SEASTAR_TEST_CASE(test_no_crash_when_a_lot_of_requests_released_which_change_reg
|
||||
});
|
||||
|
||||
auto fill_to_pressure = [&] {
|
||||
while (!gr.under_pressure()) {
|
||||
while (!gr.under_virtual_pressure()) {
|
||||
objs.emplace_back(managed_bytes(managed_bytes::initialized_later(), 1024));
|
||||
}
|
||||
};
|
||||
@@ -470,14 +470,14 @@ SEASTAR_TEST_CASE(test_no_crash_when_a_lot_of_requests_released_which_change_reg
|
||||
fill_to_pressure();
|
||||
future<> f = gr.run_when_memory_available([&, op = request_barrier.start()] {
|
||||
// Trigger group size change (Refs issue #2021)
|
||||
gr.update(-10);
|
||||
gr.update(+10);
|
||||
gr.update_virtual(-10);
|
||||
gr.update_virtual(+10);
|
||||
}, db::no_timeout);
|
||||
BOOST_REQUIRE(!f.available());
|
||||
}
|
||||
|
||||
// Release
|
||||
while (gr.under_pressure()) {
|
||||
while (gr.under_virtual_pressure()) {
|
||||
objs.pop_back();
|
||||
}
|
||||
});
|
||||
@@ -492,8 +492,8 @@ SEASTAR_TEST_CASE(test_reclaiming_runs_as_long_as_there_is_soft_pressure) {
|
||||
|
||||
bool reclaiming = false;
|
||||
region_group gr(test_name, {
|
||||
.hard_limit = hard_threshold,
|
||||
.soft_limit = soft_threshold,
|
||||
.virtual_hard_limit = hard_threshold,
|
||||
.virtual_soft_limit = soft_threshold,
|
||||
.start_reclaiming = [&] () noexcept { reclaiming = true; },
|
||||
.stop_reclaiming = [&] () noexcept { reclaiming = false; },
|
||||
});
|
||||
@@ -506,26 +506,26 @@ SEASTAR_TEST_CASE(test_reclaiming_runs_as_long_as_there_is_soft_pressure) {
|
||||
|
||||
BOOST_REQUIRE(!reclaiming);
|
||||
|
||||
while (!gr.over_soft_limit()) {
|
||||
while (!gr.over_virtual_soft_limit()) {
|
||||
objs.emplace_back(managed_bytes(managed_bytes::initialized_later(), logalloc::segment_size));
|
||||
}
|
||||
|
||||
BOOST_REQUIRE(reclaiming);
|
||||
|
||||
while (!gr.under_pressure()) {
|
||||
while (!gr.under_virtual_pressure()) {
|
||||
objs.emplace_back(managed_bytes(managed_bytes::initialized_later(), logalloc::segment_size));
|
||||
}
|
||||
|
||||
BOOST_REQUIRE(reclaiming);
|
||||
|
||||
while (gr.under_pressure()) {
|
||||
while (gr.under_virtual_pressure()) {
|
||||
objs.pop_back();
|
||||
}
|
||||
|
||||
BOOST_REQUIRE(gr.over_soft_limit());
|
||||
BOOST_REQUIRE(gr.over_virtual_soft_limit());
|
||||
BOOST_REQUIRE(reclaiming);
|
||||
|
||||
while (gr.over_soft_limit()) {
|
||||
while (gr.over_virtual_soft_limit()) {
|
||||
objs.pop_back();
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user