From 05db7d8957afd225152a09e4fe9dc6fafa62f0b0 Mon Sep 17 00:00:00 2001 From: Nadav Har'El Date: Sun, 17 Feb 2019 12:02:22 +0200 Subject: [PATCH] Materialized views: name the "batch_memory_max" constant Give the constant 1024*1024 introduced in an earlier commit a name, "batch_memory_max", and move it from view.cc to view_builder.hh. It now resides next to the pre-existing constant that controlled how many rows were read in each build step, "batch_size". Signed-off-by: Nadav Har'El Message-Id: <20190217100222.15673-1-nyh@scylladb.com> --- db/view/view.cc | 2 +- db/view/view_builder.hh | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/db/view/view.cc b/db/view/view.cc index f70fa1a2a3..4b348d2e46 100644 --- a/db/view/view.cc +++ b/db/view/view.cc @@ -1546,7 +1546,7 @@ public: _fragments_memory_usage += cr.memory_usage(*_step.base->schema()); _fragments.push_back(std::move(cr)); - if (_fragments_memory_usage > 1024*1024) { + if (_fragments_memory_usage > batch_memory_max) { // Although we have not yet completed the batch of base rows that // compact_for_query<> planned for us (view_builder::batchsize), // we've still collected enough rows to reach sizeable memory use, diff --git a/db/view/view_builder.hh b/db/view/view_builder.hh index 5ded1ea09f..34f1a37057 100644 --- a/db/view/view_builder.hh +++ b/db/view/view_builder.hh @@ -166,7 +166,12 @@ class view_builder final : public service::migration_listener::only_view_notific std::unordered_map, seastar::shared_promise<>, utils::tuple_hash> _build_notifiers; public: + // The view builder processes the base table in steps of batch_size rows. + // However, if the individual rows are large, there is no real need to + // collect batch_size of them in memory at once. Rather, as soon as we've + // collected batch_memory_max bytes, we can process the rows read so far. static constexpr size_t batch_size = 128; + static constexpr size_t batch_memory_max = 1024*1024; public: view_builder(database&, db::system_distributed_keyspace&, service::migration_manager&);