replica/table: keep track of total pre-compression file size
Every table and sstable set keeps track of the total file size of contained sstables. Due to a feature request, we also want to keep track of the hypothetical file size if Data files were uncompressed, to add a metric that shows the compression ratio of sstables. We achieve this by replacing the relevant `uint_64 bytes_on_disk` counters everywhere with a struct that contains both the actual (post-compression) size and the hypothetical pre-compression size. This patch isn't supposed to change any observable behavior. In the next patch, we will use these changes to add a new metric.
This commit is contained in:
@@ -66,6 +66,13 @@ static future<json::json_return_type> get_cf_stats(sharded<replica::database>&
|
||||
}, std::plus<int64_t>());
|
||||
}
|
||||
|
||||
static future<json::json_return_type> get_cf_stats(sharded<replica::database>& db,
|
||||
std::function<int64_t(const replica::column_family_stats&)> f) {
|
||||
return map_reduce_cf(db, int64_t(0), [f](const replica::column_family& cf) {
|
||||
return f(cf.get_stats());
|
||||
}, std::plus<int64_t>());
|
||||
}
|
||||
|
||||
static future<json::json_return_type> for_tables_on_all_shards(sharded<replica::database>& db, std::vector<table_info> tables, std::function<future<>(replica::table&)> set) {
|
||||
return do_with(std::move(tables), [&db, set] (const std::vector<table_info>& tables) {
|
||||
return db.invoke_on_all([&tables, set] (replica::database& db) {
|
||||
@@ -1066,10 +1073,14 @@ void set_column_family(http_context& ctx, routes& r, sharded<replica::database>&
|
||||
});
|
||||
|
||||
ss::get_load.set(r, [&db] (std::unique_ptr<http::request> req) {
|
||||
return get_cf_stats(db, &replica::column_family_stats::live_disk_space_used);
|
||||
return get_cf_stats(db, [](const replica::column_family_stats& stats) {
|
||||
return stats.live_disk_space_used.on_disk;
|
||||
});
|
||||
});
|
||||
ss::get_metrics_load.set(r, [&db] (std::unique_ptr<http::request> req) {
|
||||
return get_cf_stats(db, &replica::column_family_stats::live_disk_space_used);
|
||||
return get_cf_stats(db, [](const replica::column_family_stats& stats) {
|
||||
return stats.live_disk_space_used.on_disk;
|
||||
});
|
||||
});
|
||||
|
||||
ss::get_keyspaces.set(r, [&db] (const_req req) {
|
||||
|
||||
@@ -227,7 +227,7 @@ future<> run_table_tasks(replica::database& db, std::vector<table_tasks_info> ta
|
||||
// Tables will be kept in descending order.
|
||||
std::ranges::sort(table_tasks, std::greater<>(), [&] (const table_tasks_info& tti) {
|
||||
try {
|
||||
return db.find_column_family(tti.ti.id).get_stats().live_disk_space_used;
|
||||
return db.find_column_family(tti.ti.id).get_stats().live_disk_space_used.on_disk;
|
||||
} catch (const replica::no_such_column_family& e) {
|
||||
return int64_t(-1);
|
||||
}
|
||||
@@ -281,7 +281,7 @@ future<> run_keyspace_tasks(replica::database& db, std::vector<keyspace_tasks_in
|
||||
try {
|
||||
return std::accumulate(kti.table_infos.begin(), kti.table_infos.end(), int64_t(0), [&] (int64_t sum, const table_info& t) {
|
||||
try {
|
||||
sum += db.find_column_family(t.id).get_stats().live_disk_space_used;
|
||||
sum += db.find_column_family(t.id).get_stats().live_disk_space_used.on_disk;
|
||||
} catch (const replica::no_such_column_family&) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
@@ -483,7 +483,7 @@ public:
|
||||
});
|
||||
co_await add_partition(mutation_sink, "load", [this] () -> future<sstring> {
|
||||
return map_reduce_tables<int64_t>([] (replica::table& tbl) {
|
||||
return tbl.get_stats().live_disk_space_used;
|
||||
return tbl.get_stats().live_disk_space_used.on_disk;
|
||||
}).then([] (int64_t load) {
|
||||
return format("{}", load);
|
||||
});
|
||||
|
||||
@@ -224,7 +224,9 @@ public:
|
||||
|
||||
size_t live_sstable_count() const noexcept;
|
||||
uint64_t live_disk_space_used() const noexcept;
|
||||
sstables::file_size_stats live_disk_space_used_full_stats() const noexcept;
|
||||
uint64_t total_disk_space_used() const noexcept;
|
||||
sstables::file_size_stats total_disk_space_used_full_stats() const noexcept;
|
||||
|
||||
// With static sharding, i.e. vnodes, there will be only one active view.
|
||||
compaction::compaction_group_view& as_view_for_static_sharding() const;
|
||||
|
||||
@@ -397,8 +397,8 @@ struct table_stats {
|
||||
int64_t memtable_switch_count = 0;
|
||||
/** Estimated number of tasks pending for this column family */
|
||||
int64_t pending_flushes = 0;
|
||||
int64_t live_disk_space_used = 0;
|
||||
int64_t total_disk_space_used = 0;
|
||||
sstables::file_size_stats live_disk_space_used;
|
||||
sstables::file_size_stats total_disk_space_used;
|
||||
int64_t live_sstable_count = 0;
|
||||
/** Estimated number of compactions pending for this column family */
|
||||
int64_t pending_compactions = 0;
|
||||
|
||||
@@ -1344,8 +1344,8 @@ table::clone_tablet_storage(locator::tablet_id tid) {
|
||||
}
|
||||
|
||||
void table::update_stats_for_new_sstable(const sstables::shared_sstable& sst) noexcept {
|
||||
_stats.live_disk_space_used += sst->bytes_on_disk();
|
||||
_stats.total_disk_space_used += sst->bytes_on_disk();
|
||||
_stats.live_disk_space_used += sst->get_file_size_stats();
|
||||
_stats.total_disk_space_used += sst->get_file_size_stats();
|
||||
_stats.live_sstable_count++;
|
||||
}
|
||||
|
||||
@@ -1785,8 +1785,8 @@ void table::set_metrics() {
|
||||
ms::make_counter("memtable_range_tombstone_reads", _stats.memtable_range_tombstone_reads, ms::description("Number of range tombstones read from memtables"))(cf)(ks).set_skip_when_empty(),
|
||||
ms::make_counter("memtable_row_tombstone_reads", _stats.memtable_row_tombstone_reads, ms::description("Number of row tombstones read from memtables"))(cf)(ks),
|
||||
ms::make_gauge("pending_tasks", ms::description("Estimated number of tasks pending for this column family"), _stats.pending_flushes)(cf)(ks),
|
||||
ms::make_gauge("live_disk_space", ms::description("Live disk space used"), _stats.live_disk_space_used)(cf)(ks),
|
||||
ms::make_gauge("total_disk_space", ms::description("Total disk space used"), _stats.total_disk_space_used)(cf)(ks),
|
||||
ms::make_gauge("live_disk_space", ms::description("Live disk space used"), _stats.live_disk_space_used.on_disk)(cf)(ks),
|
||||
ms::make_gauge("total_disk_space", ms::description("Total disk space used"), _stats.total_disk_space_used.on_disk)(cf)(ks),
|
||||
ms::make_gauge("live_sstable", ms::description("Live sstable count"), _stats.live_sstable_count)(cf)(ks),
|
||||
ms::make_gauge("pending_compaction", ms::description("Estimated number of compactions pending for this column family"), _stats.pending_compactions)(cf)(ks),
|
||||
ms::make_gauge("pending_sstable_deletions",
|
||||
@@ -1843,9 +1843,9 @@ void table::set_metrics() {
|
||||
ms::make_counter("memtable_partition_hits", _stats.memtable_partition_hits, ms::description("Number of times a write operation was issued on an existing partition in memtables"))(cf)(ks)(node_table_metrics).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(),
|
||||
ms::make_counter("memtable_row_writes", _stats.memtable_app_stats.row_writes, ms::description("Number of row writes performed in memtables"))(cf)(ks)(node_table_metrics).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(),
|
||||
ms::make_counter("memtable_row_hits", _stats.memtable_app_stats.row_hits, ms::description("Number of rows overwritten by write operations in memtables"))(cf)(ks)(node_table_metrics).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(),
|
||||
ms::make_gauge("total_disk_space", ms::description("Total disk space used"), _stats.total_disk_space_used)(cf)(ks)(node_table_metrics).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(),
|
||||
ms::make_gauge("total_disk_space", ms::description("Total disk space used"), _stats.total_disk_space_used.on_disk)(cf)(ks)(node_table_metrics).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(),
|
||||
ms::make_gauge("live_sstable", ms::description("Live sstable count"), _stats.live_sstable_count)(cf)(ks)(node_table_metrics).aggregate({seastar::metrics::shard_label}),
|
||||
ms::make_gauge("live_disk_space", ms::description("Live disk space used"), _stats.live_disk_space_used)(cf)(ks)(node_table_metrics).aggregate({seastar::metrics::shard_label}),
|
||||
ms::make_gauge("live_disk_space", ms::description("Live disk space used"), _stats.live_disk_space_used.on_disk)(cf)(ks)(node_table_metrics).aggregate({seastar::metrics::shard_label}),
|
||||
ms::make_histogram("read_latency", ms::description("Read latency histogram"), [this] {return to_metrics_histogram(_stats.reads.histogram());})(cf)(ks)(node_table_metrics).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(),
|
||||
ms::make_histogram("write_latency", ms::description("Write latency histogram"), [this] {return to_metrics_histogram(_stats.writes.histogram());})(cf)(ks)(node_table_metrics).aggregate({seastar::metrics::shard_label}).set_skip_when_empty()
|
||||
});
|
||||
@@ -1881,6 +1881,10 @@ uint64_t compaction_group::live_disk_space_used() const noexcept {
|
||||
return _main_sstables->bytes_on_disk() + _maintenance_sstables->bytes_on_disk();
|
||||
}
|
||||
|
||||
sstables::file_size_stats compaction_group::live_disk_space_used_full_stats() const noexcept {
|
||||
return _main_sstables->get_file_size_stats() + _maintenance_sstables->get_file_size_stats();
|
||||
}
|
||||
|
||||
uint64_t storage_group::live_disk_space_used() const noexcept {
|
||||
auto cgs = const_cast<storage_group&>(*this).compaction_groups();
|
||||
return std::ranges::fold_left(cgs | std::views::transform(std::mem_fn(&compaction_group::live_disk_space_used)), uint64_t(0), std::plus{});
|
||||
@@ -1890,21 +1894,25 @@ uint64_t compaction_group::total_disk_space_used() const noexcept {
|
||||
return live_disk_space_used() + std::ranges::fold_left(_sstables_compacted_but_not_deleted | std::views::transform(std::mem_fn(&sstables::sstable::bytes_on_disk)), uint64_t(0), std::plus{});
|
||||
}
|
||||
|
||||
sstables::file_size_stats compaction_group::total_disk_space_used_full_stats() const noexcept {
|
||||
return live_disk_space_used_full_stats() + std::ranges::fold_left(_sstables_compacted_but_not_deleted | std::views::transform(std::mem_fn(&sstables::sstable::get_file_size_stats)), sstables::file_size_stats{}, std::plus{});
|
||||
}
|
||||
|
||||
void table::rebuild_statistics() {
|
||||
_stats.live_disk_space_used = 0;
|
||||
_stats.live_disk_space_used = {};
|
||||
_stats.live_sstable_count = 0;
|
||||
_stats.total_disk_space_used = 0;
|
||||
_stats.total_disk_space_used = {};
|
||||
|
||||
for_each_compaction_group([this] (const compaction_group& cg) {
|
||||
_stats.live_disk_space_used += cg.live_disk_space_used();
|
||||
_stats.total_disk_space_used += cg.total_disk_space_used();
|
||||
_stats.live_disk_space_used += cg.live_disk_space_used_full_stats();
|
||||
_stats.total_disk_space_used += cg.total_disk_space_used_full_stats();
|
||||
_stats.live_sstable_count += cg.live_sstable_count();
|
||||
});
|
||||
}
|
||||
|
||||
void table::subtract_compaction_group_from_stats(const compaction_group& cg) noexcept {
|
||||
_stats.live_disk_space_used -= cg.live_disk_space_used();
|
||||
_stats.total_disk_space_used -= cg.total_disk_space_used();
|
||||
_stats.live_disk_space_used -= cg.live_disk_space_used_full_stats();
|
||||
_stats.total_disk_space_used -= cg.total_disk_space_used_full_stats();
|
||||
_stats.live_sstable_count -= cg.live_sstable_count();
|
||||
}
|
||||
|
||||
|
||||
@@ -873,7 +873,7 @@ class tablet_sstable_set : public sstables::sstable_set_impl {
|
||||
// which provides faster lookup time.
|
||||
std::set<size_t> _sstable_set_ids;
|
||||
size_t _size = 0;
|
||||
uint64_t _bytes_on_disk = 0;
|
||||
sstables::file_size_stats _file_size_stats;
|
||||
|
||||
public:
|
||||
tablet_sstable_set(const tablet_sstable_set& o)
|
||||
@@ -882,7 +882,7 @@ public:
|
||||
, _sstable_sets(o._sstable_sets)
|
||||
, _sstable_set_ids(o._sstable_set_ids)
|
||||
, _size(o._size)
|
||||
, _bytes_on_disk(o._bytes_on_disk)
|
||||
, _file_size_stats(o._file_size_stats)
|
||||
{}
|
||||
|
||||
tablet_sstable_set(schema_ptr s, const storage_group_manager& sgm, const locator::tablet_map& tmap)
|
||||
@@ -892,7 +892,7 @@ public:
|
||||
sgm.for_each_storage_group([this] (size_t id, storage_group& sg) {
|
||||
auto set = sg.make_sstable_set();
|
||||
_size += set->size();
|
||||
_bytes_on_disk += set->bytes_on_disk();
|
||||
_file_size_stats += set->get_file_size_stats();
|
||||
_sstable_sets[id] = std::move(set);
|
||||
_sstable_set_ids.insert(id);
|
||||
});
|
||||
@@ -919,8 +919,8 @@ public:
|
||||
virtual size_t size() const noexcept override {
|
||||
return _size;
|
||||
}
|
||||
virtual uint64_t bytes_on_disk() const noexcept override {
|
||||
return _bytes_on_disk;
|
||||
virtual sstables::file_size_stats get_file_size_stats() const noexcept override {
|
||||
return _file_size_stats;
|
||||
}
|
||||
virtual selector_and_schema_t make_incremental_selector() const override;
|
||||
|
||||
|
||||
@@ -80,7 +80,7 @@ void load_broadcaster::start_broadcasting() {
|
||||
_done = _db.map_reduce0([](replica::database& db) {
|
||||
int64_t res = 0;
|
||||
db.get_tables_metadata().for_each_table([&] (table_id, lw_shared_ptr<replica::table> table) {
|
||||
res += table->get_stats().live_disk_space_used;
|
||||
res += table->get_stats().live_disk_space_used.on_disk;
|
||||
});
|
||||
return res;
|
||||
}, int64_t(0), std::plus<int64_t>()).then([this] (int64_t size) {
|
||||
|
||||
43
sstables/file_size_stats.hh
Normal file
43
sstables/file_size_stats.hh
Normal file
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright (C) 2025-present ScyllaDB
|
||||
*/
|
||||
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace sstables {
|
||||
|
||||
struct file_size_stats {
|
||||
// Actual size of files on disk.
|
||||
// In particular, for compressed sstables, this includes the post-compression size of data files.
|
||||
int64_t on_disk = 0;
|
||||
// The hypothetical size of files on disk if the data files were uncompressed.
|
||||
// (Not fully accurate. This is equivalent to `on_disk - physical_data_db_size + logical_data_db_size`.
|
||||
// For full accuracy, we would have to account for the replacement of CompressionInfo.db with CRC.db, etc.
|
||||
// We don't bother with that. This value is only used for estimation and metrics anyway).
|
||||
int64_t before_compression = 0;
|
||||
|
||||
file_size_stats& operator+=(const file_size_stats& other) {
|
||||
on_disk += other.on_disk;
|
||||
before_compression += other.before_compression;
|
||||
return *this;
|
||||
}
|
||||
file_size_stats& operator-=(const file_size_stats& other) {
|
||||
on_disk -= other.on_disk;
|
||||
before_compression -= other.before_compression;
|
||||
return *this;
|
||||
}
|
||||
friend file_size_stats operator+(const file_size_stats& lhs, const file_size_stats& rhs) {
|
||||
file_size_stats result = lhs;
|
||||
result += rhs;
|
||||
return result;
|
||||
}
|
||||
bool operator==(const file_size_stats& other) const = default;
|
||||
};
|
||||
|
||||
} // namespace sstables
|
||||
@@ -192,6 +192,11 @@ sstable_set::bytes_on_disk() const noexcept {
|
||||
return _impl->bytes_on_disk();
|
||||
}
|
||||
|
||||
file_size_stats
|
||||
sstable_set::get_file_size_stats() const noexcept {
|
||||
return _impl->get_file_size_stats();
|
||||
}
|
||||
|
||||
sstable_set::~sstable_set() = default;
|
||||
|
||||
sstable_set::incremental_selector::incremental_selector(std::unique_ptr<incremental_selector_impl> impl, const schema& s)
|
||||
@@ -323,7 +328,7 @@ static std::unordered_map<run_id, shared_sstable_run> clone_runs(const std::unor
|
||||
}
|
||||
|
||||
partitioned_sstable_set::partitioned_sstable_set(schema_ptr schema, const std::vector<shared_sstable>& unleveled_sstables, const interval_map_type& leveled_sstables,
|
||||
const lw_shared_ptr<sstable_list>& all, const std::unordered_map<run_id, shared_sstable_run>& all_runs, dht::token_range token_range, uint64_t bytes_on_disk)
|
||||
const lw_shared_ptr<sstable_list>& all, const std::unordered_map<run_id, shared_sstable_run>& all_runs, dht::token_range token_range, file_size_stats bytes_on_disk)
|
||||
: sstable_set_impl(bytes_on_disk)
|
||||
, _schema(schema)
|
||||
, _unleveled_sstables(unleveled_sstables)
|
||||
@@ -334,7 +339,7 @@ partitioned_sstable_set::partitioned_sstable_set(schema_ptr schema, const std::v
|
||||
}
|
||||
|
||||
std::unique_ptr<sstable_set_impl> partitioned_sstable_set::clone() const {
|
||||
return std::make_unique<partitioned_sstable_set>(_schema, _unleveled_sstables, _leveled_sstables, _all, _all_runs, _token_range, _bytes_on_disk);
|
||||
return std::make_unique<partitioned_sstable_set>(_schema, _unleveled_sstables, _leveled_sstables, _all, _all_runs, _token_range, _file_size_stats);
|
||||
}
|
||||
|
||||
std::vector<shared_sstable> partitioned_sstable_set::select(const dht::partition_range& range) const {
|
||||
@@ -376,11 +381,11 @@ bool partitioned_sstable_set::insert(shared_sstable sst) {
|
||||
// sst is already in the set, no further handling is required
|
||||
return false;
|
||||
}
|
||||
auto n = sst->bytes_on_disk();
|
||||
add_bytes_on_disk(n);
|
||||
auto size_stats = sst->get_file_size_stats();
|
||||
add_file_size_stats(size_stats);
|
||||
auto undo_all_insert = defer([&] () {
|
||||
_all->erase(sst);
|
||||
sub_bytes_on_disk(n);
|
||||
sub_file_size_stats(size_stats);
|
||||
});
|
||||
|
||||
auto maybe_insert_run_fragment = [this] (const shared_sstable& sst) mutable {
|
||||
@@ -420,7 +425,7 @@ bool partitioned_sstable_set::erase(shared_sstable sst) {
|
||||
}
|
||||
auto ret = _all->erase(sst) != 0;
|
||||
if (ret) {
|
||||
sub_bytes_on_disk(sst->bytes_on_disk());
|
||||
sub_file_size_stats(sst->get_file_size_stats());
|
||||
}
|
||||
if (store_as_unleveled(sst)) {
|
||||
_unleveled_sstables.erase(std::remove(_unleveled_sstables.begin(), _unleveled_sstables.end(), sst), _unleveled_sstables.end());
|
||||
@@ -557,7 +562,7 @@ bool time_series_sstable_set::insert(shared_sstable sst) {
|
||||
auto min_pos = sst->min_position();
|
||||
auto max_pos_reversed = sst->max_position().reversed();
|
||||
_sstables->emplace(std::move(min_pos), sst);
|
||||
add_bytes_on_disk(sst->bytes_on_disk());
|
||||
add_file_size_stats(sst->get_file_size_stats());
|
||||
_sstables_reversed->emplace(std::move(max_pos_reversed), std::move(sst));
|
||||
} catch (...) {
|
||||
erase(sst);
|
||||
@@ -576,7 +581,7 @@ bool time_series_sstable_set::erase(shared_sstable sst) {
|
||||
found = it != last;
|
||||
if (found) {
|
||||
_sstables->erase(it);
|
||||
sub_bytes_on_disk(sst->bytes_on_disk());
|
||||
sub_file_size_stats(sst->get_file_size_stats());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1179,9 +1184,9 @@ compound_sstable_set::size() const noexcept {
|
||||
return std::ranges::fold_left(_sets | std::views::transform(std::mem_fn(&sstable_set::size)), size_t(0), std::plus{});
|
||||
}
|
||||
|
||||
uint64_t
|
||||
compound_sstable_set::bytes_on_disk() const noexcept {
|
||||
return std::ranges::fold_left(_sets | std::views::transform(std::mem_fn(&sstable_set::bytes_on_disk)), uint64_t(0), std::plus{});
|
||||
file_size_stats
|
||||
compound_sstable_set::get_file_size_stats() const noexcept {
|
||||
return std::ranges::fold_left(_sets | std::views::transform(std::mem_fn(&sstable_set::get_file_size_stats)), file_size_stats{}, std::plus{});
|
||||
}
|
||||
|
||||
class compound_sstable_set::incremental_selector : public incremental_selector_impl {
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#include "readers/mutation_reader.hh"
|
||||
#include "sstables/progress_monitor.hh"
|
||||
#include "sstables/types_fwd.hh"
|
||||
#include "sstables/file_size_stats.hh"
|
||||
#include "shared_sstable.hh"
|
||||
#include "dht/ring_position.hh"
|
||||
#include <seastar/core/shared_ptr.hh>
|
||||
@@ -77,10 +78,10 @@ const sstable_predicate& default_sstable_predicate();
|
||||
|
||||
class sstable_set_impl {
|
||||
protected:
|
||||
uint64_t _bytes_on_disk = 0;
|
||||
file_size_stats _file_size_stats;
|
||||
|
||||
// for cloning
|
||||
explicit sstable_set_impl(uint64_t bytes_on_disk) noexcept : _bytes_on_disk(bytes_on_disk) {}
|
||||
explicit sstable_set_impl(file_size_stats bytes_on_disk) noexcept : _file_size_stats(bytes_on_disk) {}
|
||||
public:
|
||||
sstable_set_impl() = default;
|
||||
sstable_set_impl(const sstable_set_impl&) = default;
|
||||
@@ -96,14 +97,17 @@ public:
|
||||
// Return true iff sst was erased
|
||||
virtual bool erase(shared_sstable sst) = 0;
|
||||
virtual size_t size() const noexcept = 0;
|
||||
virtual uint64_t bytes_on_disk() const noexcept {
|
||||
return _bytes_on_disk;
|
||||
uint64_t bytes_on_disk() const noexcept {
|
||||
return get_file_size_stats().on_disk;
|
||||
}
|
||||
uint64_t add_bytes_on_disk(uint64_t delta) noexcept {
|
||||
return _bytes_on_disk += delta;
|
||||
virtual file_size_stats get_file_size_stats() const noexcept {
|
||||
return _file_size_stats;
|
||||
}
|
||||
uint64_t sub_bytes_on_disk(uint64_t delta) noexcept {
|
||||
return _bytes_on_disk -= delta;
|
||||
void add_file_size_stats(const file_size_stats& delta) noexcept {
|
||||
_file_size_stats += delta;
|
||||
}
|
||||
void sub_file_size_stats(const file_size_stats& delta) noexcept {
|
||||
_file_size_stats -= delta;
|
||||
}
|
||||
using selector_and_schema_t = std::tuple<std::unique_ptr<incremental_selector_impl>, const schema&>;
|
||||
virtual selector_and_schema_t make_incremental_selector() const = 0;
|
||||
@@ -163,6 +167,7 @@ public:
|
||||
bool erase(shared_sstable sst);
|
||||
size_t size() const noexcept;
|
||||
uint64_t bytes_on_disk() const noexcept;
|
||||
file_size_stats get_file_size_stats() const noexcept;
|
||||
|
||||
// Used to incrementally select sstables from sstable set using ring-position.
|
||||
// sstable set must be alive during the lifetime of the selector.
|
||||
|
||||
@@ -59,7 +59,7 @@ public:
|
||||
const lw_shared_ptr<sstable_list>& all,
|
||||
const std::unordered_map<run_id, shared_sstable_run>& all_runs,
|
||||
dht::token_range token_range,
|
||||
uint64_t bytes_on_disk);
|
||||
file_size_stats bytes_on_disk);
|
||||
|
||||
virtual std::unique_ptr<sstable_set_impl> clone() const override;
|
||||
virtual std::vector<shared_sstable> select(const dht::partition_range& range) const override;
|
||||
@@ -139,7 +139,7 @@ public:
|
||||
virtual bool insert(shared_sstable sst) override;
|
||||
virtual bool erase(shared_sstable sst) override;
|
||||
virtual size_t size() const noexcept override;
|
||||
virtual uint64_t bytes_on_disk() const noexcept override;
|
||||
virtual file_size_stats get_file_size_stats() const noexcept override;
|
||||
virtual sstable_set_impl::selector_and_schema_t make_incremental_selector() const override;
|
||||
|
||||
virtual mutation_reader create_single_key_sstable_reader(
|
||||
|
||||
@@ -2355,7 +2355,7 @@ uint64_t sstable::ondisk_data_size() const {
|
||||
return _data_file_size;
|
||||
}
|
||||
|
||||
uint64_t sstable::bytes_on_disk() const {
|
||||
file_size_stats sstable::get_file_size_stats() const {
|
||||
if (!_metadata_size_on_disk) {
|
||||
on_internal_error(sstlog, "On-disk size of sstable metadata was not set");
|
||||
}
|
||||
@@ -2365,7 +2365,16 @@ uint64_t sstable::bytes_on_disk() const {
|
||||
if (!_index_file_size && !_partitions_file_size) {
|
||||
on_internal_error(sstlog, "On-disk size of sstable index was not set");
|
||||
}
|
||||
return _metadata_size_on_disk + _data_file_size + _index_file_size + _partitions_file_size + _rows_file_size;
|
||||
uint64_t size_without_data = _metadata_size_on_disk + _index_file_size + _partitions_file_size + _rows_file_size;
|
||||
|
||||
file_size_stats stats;
|
||||
stats.on_disk = size_without_data + _data_file_size;
|
||||
stats.before_compression = size_without_data + data_size();
|
||||
return stats;
|
||||
}
|
||||
|
||||
uint64_t sstable::bytes_on_disk() const {
|
||||
return get_file_size_stats().on_disk;
|
||||
}
|
||||
|
||||
uint64_t sstable::filter_size() const {
|
||||
|
||||
@@ -46,6 +46,7 @@
|
||||
#include "dht/decorated_key.hh"
|
||||
#include "service/session.hh"
|
||||
#include "sstables/trie/bti_index.hh"
|
||||
#include "sstables/file_size_stats.hh"
|
||||
|
||||
#include <seastar/util/optimized_optional.hh>
|
||||
|
||||
@@ -397,6 +398,7 @@ public:
|
||||
|
||||
// Returns the total bytes of all components.
|
||||
uint64_t bytes_on_disk() const;
|
||||
file_size_stats get_file_size_stats() const;
|
||||
|
||||
const partition_key& get_first_partition_key() const;
|
||||
const partition_key& get_last_partition_key() const;
|
||||
|
||||
@@ -5410,7 +5410,7 @@ future<> run_controller_test(compaction::compaction_strategy_type compaction_str
|
||||
available_space -= std::min(available_space, uint64_t(tier_size));
|
||||
}
|
||||
|
||||
auto table_size = t->get_stats().live_disk_space_used;
|
||||
auto table_size = t->get_stats().live_disk_space_used.on_disk;
|
||||
testlog.debug("T{}: {} tiers, with total size={}", t_idx, tiers, utils::pretty_printed_data_size(table_size));
|
||||
tables.push_back(t);
|
||||
tables_total_size += table_size;
|
||||
|
||||
@@ -241,7 +241,7 @@ void test_main_thread(cql_test_env& env) {
|
||||
|
||||
env.local_db().flush_all_memtables().get();
|
||||
|
||||
testlog.info("Live disk space used: {}", tab.get_stats().live_disk_space_used);
|
||||
testlog.info("Live disk space used: {}", tab.get_stats().live_disk_space_used.on_disk);
|
||||
testlog.info("Live sstables: {}", tab.get_stats().live_sstable_count);
|
||||
|
||||
testlog.info("Preparing dummy cache");
|
||||
|
||||
@@ -148,7 +148,7 @@ static future<> test_basic_operations(app_template& app) {
|
||||
testlog.info("Size of canonical mutations: {:.6f} [MiB]", double(cm_size) / MiB);
|
||||
|
||||
auto&& tablets_table = e.local_db().find_column_family(db::system_keyspace::tablets());
|
||||
testlog.info("Disk space used by system.tablets: {:.6f} [MiB]", double(tablets_table.get_stats().live_disk_space_used) / MiB);
|
||||
testlog.info("Disk space used by system.tablets: {:.6f} [MiB]", double(tablets_table.get_stats().live_disk_space_used.on_disk) / MiB);
|
||||
|
||||
locator::tablet_metadata_change_hint hint;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user