From a1c86786ca878509f2acfa1fb4a97b46164cfbe3 Mon Sep 17 00:00:00 2001 From: Jan Ciolek Date: Tue, 14 Mar 2023 18:40:39 +0100 Subject: [PATCH] db/view/view.cc: rate limit view update error messages When propagating a view update to a paired view replica fails, there is an error message. This message is printed for every mutation, which causes log spam when some node goes down. This isn't a fatal error - it's normal that a remote view replica goes down, it'll hopefully receive the updates later through hints. I'm unsure if the error message should be printed at all, but for now we can just rate limit it and that will improve the situation with log spamming. Signed-off-by: Jan Ciolek Closes #13175 --- db/view/view.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/db/view/view.cc b/db/view/view.cc index c0f56b1805..879ec95c16 100644 --- a/db/view/view.cc +++ b/db/view/view.cc @@ -1722,8 +1722,12 @@ future<> mutate_MV( auto ep = f.get_exception(); tracing::trace(tr_state, "Failed to apply view update for {} and {} remote endpoints", *target_endpoint, updates_pushed_remote); - vlogger.error("Error applying view update to {} (view: {}.{}, base token: {}, view token: {}): {}", - *target_endpoint, s->ks_name(), s->cf_name(), base_token, view_token, ep); + + // Printing an error on every failed view mutation would cause log spam, so a rate limit is needed. + static thread_local logger::rate_limit view_update_error_rate_limit(std::chrono::seconds(4)); + vlogger.log(log_level::error, view_update_error_rate_limit, + "Error applying view update to {} (view: {}.{}, base token: {}, view token: {}): {}", + *target_endpoint, s->ks_name(), s->cf_name(), base_token, view_token, ep); return apply_update_synchronously ? make_exception_future<>(std::move(ep)) : make_ready_future<>(); } tracing::trace(tr_state, "Successfully applied view update for {} and {} remote endpoints",