bahman/scylla
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

mutation_partition: make static_row optional to reduce memory footprint

Browse Source 
The static row can be rare: many tables don't have them, and tables
that do will often have mutations without them (if the static row
is rarely updated, it may be present in the cache and in readers,
but absent in memtable mutations). However, it always consumes ~100
bytes of memory, even if it not present, due to row's overhead.

Change it to be optional by using lazy_row instead of row. Some call
sites treewide were adjusted to deal with the extra indirection.

perf_simple_query appears to improve by 2%, from 163krps to 165 krps,
though it's hard to be sure due to noisy measurements.

memory_footprint comparisons (before/after):

mutation footprint:		       mutation footprint:
 - in cache:	 1096		        - in cache:	992
 - in memtable:	 854		        - in memtable:	750
 - in sstable:	 351		        - in sstable:	351
 - frozen:	 540		        - frozen:	540
 - canonical:	 827		        - canonical:	827
 - query result: 342		        - query result: 342

 sizeof(cache_entry) = 112	        sizeof(cache_entry) = 112
 -- sizeof(decorated_key) = 36	        -- sizeof(decorated_key) = 36
 -- sizeof(cache_link_type) = 32        -- sizeof(cache_link_type) = 32
 -- sizeof(mutation_partition) = 200    -- sizeof(mutation_partition) = 96
 -- -- sizeof(_static_row) = 112        -- -- sizeof(_static_row) = 8
 -- -- sizeof(_rows) = 24	        -- -- sizeof(_rows) = 24
 -- -- sizeof(_row_tombstones) = 40     -- -- sizeof(_row_tombstones) = 40

 sizeof(rows_entry) = 232	        sizeof(rows_entry) = 232
 sizeof(lru_link_type) = 16	        sizeof(lru_link_type) = 16
 sizeof(deletable_row) = 168	        sizeof(deletable_row) = 168
 sizeof(row) = 112		        sizeof(row) = 112
 sizeof(atomic_cell_or_collection) = 8  sizeof(atomic_cell_or_collection) = 8

Tests: unit (dev)
This commit is contained in:
Avi Kivity
2019-10-04 15:04:49 +03:00
parent 88613e6882
commit acc433b286
13 changed files with 32 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cell_locking.hh
View File

@@ -68,7 +68,7 @@ public:
public:
explicit iterator(const mutation_partition& mp)
: _mp(mp)
, _current(position_in_partition_view(position_in_partition_view::static_row_tag_t()), mp.static_row())
, _current(position_in_partition_view(position_in_partition_view::static_row_tag_t()), mp.static_row().get())
{ }
iterator(const mutation_partition& mp, mutation_partition::rows_type::const_iterator it)

4
converting_mutation_partition_applier.hh
View File

@@ -100,7 +100,7 @@ public:
const column_mapping_entry& col = _visited_column_mapping.static_column_at(id);
const column_definition* def = _p_schema.get_column_definition(col.name());
if (def) {
accept_cell(_p._static_row, column_kind::static_column, *def, col.type(), cell);
accept_cell(_p._static_row.maybe_create(), column_kind::static_column, *def, col.type(), cell);
}
}
@@ -108,7 +108,7 @@ public:
const column_mapping_entry& col = _visited_column_mapping.static_column_at(id);
const column_definition* def = _p_schema.get_column_definition(col.name());
if (def) {
accept_cell(_p._static_row, column_kind::static_column, *def, col.type(), collection);
accept_cell(_p._static_row.maybe_create(), column_kind::static_column, *def, col.type(), collection);
}
}

2
flat_mutation_reader.cc
View File

@@ -419,7 +419,7 @@ flat_mutation_reader_from_mutations(std::vector<mutation> mutations, const dht::
if (!_static_row_done) {
_static_row_done = true;
if (!_cur->partition().static_row().empty()) {
push_mutation_fragment(static_row(std::move(_cur->partition().static_row())));
push_mutation_fragment(static_row(std::move(_cur->partition().static_row().get_existing())));
}
}
auto mfopt = read_next();

2
memtable.cc
View File

@@ -104,7 +104,7 @@ void memtable::memtable_encoding_stats_collector::update(const ::schema& s, cons
void memtable::memtable_encoding_stats_collector::update(const ::schema& s, const mutation_partition& mp) {
update(mp.partition_tombstone());
update(s, mp.static_row(), column_kind::static_column);
update(s, mp.static_row().get(), column_kind::static_column);
for (auto&& row_entry : mp.clustered_rows()) {
update(s, row_entry.row());
}

12
mutation_partition.cc
View File

@@ -876,13 +876,13 @@ mutation_partition::query_compacted(query::result::partition_writer& pw, const s
if (!slice.static_columns.empty()) {
if (pw.requested_result()) {
get_compacted_row_slice(s, slice, column_kind::static_column, static_row(), slice.static_columns, static_cells_wr);
get_compacted_row_slice(s, slice, column_kind::static_column, static_row().get(), slice.static_columns, static_cells_wr);
}
if (pw.requested_digest()) {
auto pt = partition_tombstone();
pw.digest().feed_hash(pt);
max_ts.update(pt.timestamp);
pw.digest().feed_hash(static_row(), s, column_kind::static_column, slice.static_columns, max_ts);
pw.digest().feed_hash(static_row().get(), s, column_kind::static_column, slice.static_columns, max_ts);
}
}
@@ -937,7 +937,7 @@ mutation_partition::query_compacted(query::result::partition_writer& pw, const s
// rows, or return nothing, since cql does not allow "is null".
if (row_count == 0
&& (has_ck_selector(pw.ranges())
|| !has_any_live_data(s, column_kind::static_column, static_row()))) {
|| !has_any_live_data(s, column_kind::static_column, static_row().get()))) {
pw.retract();
} else {
pw.row_count() += row_count ? : 1;
@@ -1021,7 +1021,7 @@ operator<<(std::ostream& os, const mutation_partition::printer& p) {
if (!mp._row_tombstones.empty()) {
os << "\n range_tombstones: {" << ::join(",", prefixed("\n ", mp._row_tombstones)) << "},";
}
os << "\n static: cont=" << int(mp._static_row_continuous) << " " << row::printer(p._schema, column_kind::static_column, mp._static_row) << ",";
os << "\n static: cont=" << int(mp._static_row_continuous) << " " << lazy_row::printer(p._schema, column_kind::static_column, mp._static_row) << ",";
auto add_printer = [&] (const auto& re) {
return rows_entry::printer(p._schema, re);
};
@@ -1465,7 +1465,7 @@ row::is_live(const schema& s, column_kind kind, tombstone base_tombstone, gc_clo
bool
mutation_partition::is_static_row_live(const schema& s, gc_clock::time_point query_time) const {
check_schema(s);
return has_any_live_data(s, column_kind::static_column, static_row(), _tombstone, query_time);
return has_any_live_data(s, column_kind::static_column, static_row().get(), _tombstone, query_time);
}
size_t
@@ -2300,7 +2300,7 @@ public:
}
void consume(tombstone) { }
stop_iteration consume(static_row&& sr, tombstone, bool) {
_mutation->partition().static_row() = std::move(sr.cells());
_mutation->partition().static_row().maybe_create() = std::move(sr.cells());
return stop_iteration::no;
}
stop_iteration consume(clustering_row&& cr, row_tombstone, bool) {

6
mutation_partition.hh
View File

@@ -1200,7 +1200,7 @@ public:
friend class size_calculator;
private:
tombstone _tombstone;
row _static_row;
lazy_row _static_row;
bool _static_row_continuous = true;
rows_type _rows;
// Contains only strict prefixes so that we don't have to lookup full keys
@@ -1409,8 +1409,8 @@ public:
deletable_row& clustered_row(const schema& s, position_in_partition_view pos, is_dummy, is_continuous);
public:
tombstone partition_tombstone() const { return _tombstone; }
row& static_row() { return _static_row; }
const row& static_row() const { return _static_row; }
lazy_row& static_row() { return _static_row; }
const lazy_row& static_row() const { return _static_row; }
// return a set of rows_entry where each entry represents a CQL row sharing the same clustering key.
const rows_type& clustered_rows() const { return _rows; }
const range_tombstone_list& row_tombstones() const { return _row_tombstones; }

2
mutation_partition_serializer.cc
View File

@@ -202,7 +202,7 @@ template<typename Writer>
void mutation_partition_serializer::write_serialized(Writer&& writer, const schema& s, const mutation_partition& mp)
{
auto srow_writer = std::move(writer).write_tomb(mp.partition_tombstone()).start_static_row();
auto row_tombstones = write_row_cells(std::move(srow_writer), mp.static_row(), s, column_kind::static_column).end_static_row().start_range_tombstones();
auto row_tombstones = write_row_cells(std::move(srow_writer), mp.static_row().get(), s, column_kind::static_column).end_static_row().start_range_tombstones();
write_tombstones(s, row_tombstones, mp.row_tombstones());
auto clustering_rows = std::move(row_tombstones).end_range_tombstones().start_rows();
for (auto&& cr : mp.non_dummy_rows()) {

4
partition_builder.hh
View File

@@ -48,12 +48,12 @@ public:
}
void accept_static_cell(column_id id, atomic_cell&& cell) {
row& r = _partition.static_row();
row& r = _partition.static_row().maybe_create();
r.append_cell(id, atomic_cell_or_collection(std::move(cell)));
}
virtual void accept_static_cell(column_id id, collection_mutation_view collection) override {
row& r = _partition.static_row();
row& r = _partition.static_row().maybe_create();
auto& ctype = *static_pointer_cast<const collection_type_impl>(_schema.static_column_at(id).type);
r.append_cell(id, atomic_cell_or_collection(collection_mutation(ctype, collection)));
}

4
partition_version.cc
View File

@@ -141,7 +141,7 @@ inline Result squashed(const partition_version_ref& v, Map&& map, Reduce&& reduc
if (digest_requested) {
mp.static_row().prepare_hash(*_schema, column_kind::static_column);
}
return mp.static_row();
return mp.static_row().get();
},
[this] (const row& r) { return row(*_schema, column_kind::static_column, r); },
[this] (row& a, const row& b) { a.apply(*_schema, column_kind::static_column, b); }));
@@ -523,7 +523,7 @@ coroutine partition_entry::apply_to_incomplete(const schema& s,
+ current->partition().static_row().external_memory_usage(s, column_kind::static_column);
dst.partition().apply(current->partition().partition_tombstone());
if (static_row_continuous) {
row& static_row = dst.partition().static_row();
lazy_row& static_row = dst.partition().static_row();
if (can_move) {
static_row.apply(s, column_kind::static_column,
std::move(current->partition().static_row()));

2
tests/multishard_mutation_query_test.cc
View File

@@ -786,7 +786,7 @@ static void validate_result(size_t i, const mutation& result_mut, const expected
const auto wrapper = with_schema_wrapper{schema};
BOOST_REQUIRE_EQUAL(result_mut.partition().static_row().empty(), !expected_part.has_static_row);
validate_static_row(schema, expected_part.dkey.key(), result_mut.partition().static_row());
validate_static_row(schema, expected_part.dkey.key(), result_mut.partition().static_row().get());
const auto& res_rows = result_mut.partition().clustered_rows();
auto res_it = res_rows.begin();

2
tests/mutation_source_test.cc
View File

@@ -1893,7 +1893,7 @@ public:
m.partition().set_static_row_continuous(_bool_dist(_gen));
set_random_cells(m.partition().static_row(), column_kind::static_column);
set_random_cells(m.partition().static_row().maybe_create(), column_kind::static_column);
auto row_count_dist = [&] (auto& gen) {
static thread_local std::normal_distribution<> dist(32, 1.5);

12
tests/mutation_test.cc
View File

@@ -237,7 +237,7 @@ SEASTAR_TEST_CASE(test_map_mutations) {
mt->apply(m2o);
auto p = get_partition(*mt, key);
row& r = p.static_row();
lazy_row& r = p.static_row();
auto i = r.find_cell(column.id);
BOOST_REQUIRE(i);
auto cell = i->as_collection_mutation();
@@ -274,7 +274,7 @@ SEASTAR_TEST_CASE(test_set_mutations) {
mt->apply(m2o);
auto p = get_partition(*mt, key);
row& r = p.static_row();
lazy_row& r = p.static_row();
auto i = r.find_cell(column.id);
BOOST_REQUIRE(i);
auto cell = i->as_collection_mutation();
@@ -312,7 +312,7 @@ SEASTAR_TEST_CASE(test_list_mutations) {
mt->apply(m2o);
auto p = get_partition(*mt, key);
row& r = p.static_row();
lazy_row& r = p.static_row();
auto i = r.find_cell(column.id);
BOOST_REQUIRE(i);
auto cell = i->as_collection_mutation();
@@ -1007,7 +1007,7 @@ SEASTAR_TEST_CASE(test_large_blobs) {
mt->apply(std::move(m));
auto p = get_partition(*mt, key);
row& r = p.static_row();
lazy_row& r = p.static_row();
auto i = r.find_cell(s1_col.id);
BOOST_REQUIRE(i);
auto cell = i->as_atomic_cell(s1_col);
@@ -1020,7 +1020,7 @@ SEASTAR_TEST_CASE(test_large_blobs) {
mt->apply(std::move(m2));
auto p2 = get_partition(*mt, key);
row& r2 = p2.static_row();
lazy_row& r2 = p2.static_row();
auto i2 = r2.find_cell(s1_col.id);
BOOST_REQUIRE(i2);
auto cell2 = i2->as_atomic_cell(s1_col);
@@ -2444,7 +2444,7 @@ partition_summary summarize_mutation(const mutation& m) {
m.partition().partition_tombstone(),
m.partition().static_row().empty() ?
std::nullopt :
std::optional(static_row_summary{summarize_row(schema, column_kind::static_column, m.partition().static_row())}),
std::optional(static_row_summary{summarize_row(schema, column_kind::static_column, m.partition().static_row().get())}),
std::move(clustering_fragments));
}

10
tests/sstable_mutation_test.cc
View File

@@ -164,7 +164,7 @@ SEASTAR_THREAD_TEST_CASE(complex_sst1_k1) {
generate_clustered<1>(env, "key1").then([] (auto&& mutation) {
auto s = complex_schema();
auto& sr = mutation.partition().static_row();
auto& sr = mutation.partition().static_row().get();
match_live_cell(sr, *s, "static_obj", data_value(to_bytes("static_value")));
auto row1 = clustered_row(mutation, *s, {"cl1.1", "cl2.1"});
@@ -195,7 +195,7 @@ SEASTAR_THREAD_TEST_CASE(complex_sst1_k2) {
generate_clustered<1>(env, "key2").then([] (auto&& mutation) {
auto s = complex_schema();
auto& sr = mutation.partition().static_row();
auto& sr = mutation.partition().static_row().get();
match_live_cell(sr, *s, "static_obj", data_value(to_bytes("static_value")));
auto static_set = match_collection(sr, *s, "static_collection", tombstone(deletion_time{1431451390, 1431451390225257l}));
match_collection_element<status::live>(static_set.cells[0], to_bytes("1"), bytes_opt{});
@@ -248,7 +248,7 @@ SEASTAR_THREAD_TEST_CASE(complex_sst2_k2) {
generate_clustered<2>(env, "key2").then([] (auto&& mutation) {
auto s = complex_schema();
auto& sr = mutation.partition().static_row();
auto& sr = mutation.partition().static_row().get();
match_dead_cell(sr, *s, "static_obj");
auto static_set = match_collection(sr, *s, "static_collection", tombstone(deletion_time{0, api::missing_timestamp}));
match_collection_element<status::dead>(static_set.cells[0], to_bytes("1"), bytes_opt{});
@@ -279,7 +279,7 @@ SEASTAR_THREAD_TEST_CASE(complex_sst2_k3) {
generate_clustered<2>(env, "key3").then([] (auto&& mutation) {
auto s = complex_schema();
auto& sr = mutation.partition().static_row();
auto& sr = mutation.partition().static_row().get();
match_expiring_cell(sr, *s, "static_obj", data_value(to_bytes("static_value_3")), 1431451394597062l, 1431537794);
auto row1 = clustered_row(mutation, *s, {"tcl1.1", "tcl2.1"});
@@ -321,7 +321,7 @@ SEASTAR_THREAD_TEST_CASE(complex_sst3_k2) {
generate_clustered<3>(env, "key2").then([] (auto&& mutation) {
auto s = complex_schema();
auto& sr = mutation.partition().static_row();
auto& sr = mutation.partition().static_row().get();
match_live_cell(sr, *s, "static_obj", data_value(to_bytes("final_static")));
auto row = clustered_row(mutation, *s, {"kcl1.1", "kcl2.1"});