diff --git a/service/storage_proxy.cc b/service/storage_proxy.cc index 4e2cbe0d24..16ae4c889a 100644 --- a/service/storage_proxy.cc +++ b/service/storage_proxy.cc @@ -1672,6 +1672,14 @@ void storage_proxy_stats::stats::register_stats() { sm::description("number read requests failed due to an \"unavailable\" error"), {storage_proxy_stats::current_scheduling_group_label()}), + sm::make_total_operations("read_rate_limited", read_rate_limited_by_replicas._count, + sm::description("number of read requests which were rejected by replicas because rate limit for the partition was reached."), + {storage_proxy_stats::current_scheduling_group_label(), storage_proxy_stats::rejected_by_coordinator_label(false)}), + + sm::make_total_operations("read_rate_limited", read_rate_limited_by_coordinator._count, + sm::description("number of read requests which were rejected directly on the coordinator because rate limit for the partition was reached."), + {storage_proxy_stats::current_scheduling_group_label(), storage_proxy_stats::rejected_by_coordinator_label(true)}), + sm::make_total_operations("range_timeouts", range_slice_timeouts._count, sm::description("number of range read operations failed due to a timeout"), {storage_proxy_stats::current_scheduling_group_label()}), @@ -4251,6 +4259,14 @@ void storage_proxy::handle_read_error(std::variant([&] (const auto& ex) { + slogger.debug("Read was rate limited"); + if (ex.rejected_by_coordinator) { + get_stats().read_rate_limited_by_coordinator.mark(); + } else { + get_stats().read_rate_limited_by_replicas.mark(); + } + return bo::success(); }), utils::result_catch_dots([&] (auto&& handle) { slogger.debug("Error during read query {}", handle.as_inner()); return bo::success(); diff --git a/service/storage_proxy_stats.hh b/service/storage_proxy_stats.hh index ade1a901c5..fcecbd508b 100644 --- a/service/storage_proxy_stats.hh +++ b/service/storage_proxy_stats.hh @@ -127,6 +127,8 @@ struct stats : public write_stats { seastar::metrics::metric_groups _metrics; utils::timed_rate_moving_average read_timeouts; utils::timed_rate_moving_average read_unavailables; + utils::timed_rate_moving_average read_rate_limited_by_replicas; + utils::timed_rate_moving_average read_rate_limited_by_coordinator; utils::timed_rate_moving_average range_slice_timeouts; utils::timed_rate_moving_average range_slice_unavailables;