bahman/scylla
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Merge 'utils: error injection: add a string-to-string map of injection's parameters' from Mikołaj Grzebieluch

Browse Source 
Add `parameters` map to `injection_shared_data`. Now tests can attach
string data to injections that can be read in injected code via
`injection_handler`.

Closes #14521

Closes #14608

* github.com:scylladb/scylladb:
  tests: add a `parameters` argument to code that enables injections
  api/error_injection: add passing injection's parameters to enable endpoint
  tests: utils: error injection: add test for injection's parameters
  utils: error injection: add a string-to-string map of injection's parameters
  utils: error injection: rename received_messages_counter to injection_shared_data
This commit is contained in:
Kamil Braun
2023-07-13 11:52:15 +02:00
parent a871de33e6 382d797d81
commit a2fe63349d
5 changed files with 108 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
api/api-doc/error_injection.json
View File

@@ -34,6 +34,14 @@
"allowMultiple":false,
"type":"boolean",
"paramType":"query"
},
{
"name":"parameters",
"description":"dict of parameters to pass to the injection (json format)",
"required":false,
"allowMultiple":false,
"type":"dict",
"paramType":"body"
}
]
},
@@ -110,5 +118,15 @@
}
]
}
]
],
"components":{
"schemas": {
"dict": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}

27
api/error_injection.cc
View File

@@ -12,7 +12,9 @@
#include <seastar/http/exception.hh>
#include "log.hh"
#include "utils/error_injection.hh"
#include "utils/rjson.hh"
#include <seastar/core/future-util.hh>
#include <seastar/util/short_streams.hh>
namespace api {
using namespace seastar::httpd;
@@ -24,10 +26,27 @@ void set_error_injection(http_context& ctx, routes& r) {
hf::enable_injection.set(r, [](std::unique_ptr<request> req) {
sstring injection = req->param["injection"];
bool one_shot = req->get_query_param("one_shot") == "True";
auto& errinj = utils::get_local_injector();
return errinj.enable_on_all(injection, one_shot).then([] {
return make_ready_future<json::json_return_type>(json::json_void());
});
auto params = req->content;
const size_t max_params_size = 1024 * 1024;
if (params.size() > max_params_size) {
// This is a hard limit, because we don't want to allocate
// too much memory or block the thread for too long.
throw httpd::bad_param_exception(format("Injection parameters are too long, max length is {}", max_params_size));
}
try {
auto parameters = params.empty()
? utils::error_injection_parameters{}
: rjson::parse_to_map<utils::error_injection_parameters>(params);
auto& errinj = utils::get_local_injector();
return errinj.enable_on_all(injection, one_shot, std::move(parameters)).then([] {
return make_ready_future<json::json_return_type>(json::json_void());
});
} catch (const rjson::error& e) {
throw httpd::bad_param_exception(format("Failed to parse injections parameters: {}", e.what()));
}
});
hf::get_enabled_injections_on_all.set(r, [](std::unique_ptr<request> req) {

16
test/boost/error_injection_test.cc
View File

@@ -333,6 +333,22 @@ SEASTAR_TEST_CASE(test_inject_message) {
}
}
SEASTAR_TEST_CASE(test_inject_with_parameters) {
utils::error_injection<true> errinj;
errinj.enable("injection", false, { { "x", "42" } });
auto f = errinj.inject_with_handler("injection", [] (auto& handler) {
auto x = handler.get("x");
auto y = handler.get("y");
BOOST_REQUIRE(x && *x == "42");
BOOST_REQUIRE(!y);
return make_ready_future<>();
});
BOOST_REQUIRE_NO_THROW(co_await std::move(f));
}
// Test error injection CQL API
// NOTE: currently since functions can't get terminals an auxiliary table
// with error injection names and one shot parameters

12
test/pylib/rest_client.py
View File

@@ -196,13 +196,13 @@ class ScyllaRESTAPIClient():
assert(type(data) == list)
return data
async def enable_injection(self, node_ip: str, injection: str, one_shot: bool) -> None:
async def enable_injection(self, node_ip: str, injection: str, one_shot: bool, parameters: dict[str, Any] = {}) -> None:
"""Enable error injection named `injection` on `node_ip`. Depending on `one_shot`,
the injection will be executed only once or every time the process passes the injection point.
Note: this only has an effect in specific build modes: debug,dev,sanitize.
"""
await self.client.post(f"/v2/error_injection/injection/{injection}",
host=node_ip, params={"one_shot": str(one_shot)})
host=node_ip, params={"one_shot": str(one_shot)}, json={ key: str(value) for key, value in parameters.items() })
async def disable_injection(self, node_ip: str, injection: str) -> None:
await self.client.delete(f"/v2/error_injection/injection/{injection}", host=node_ip)
@@ -237,13 +237,13 @@ class InjectionHandler():
await self.api.message_injection(self.node_ip, self.injection)
@asynccontextmanager
async def inject_error(api: ScyllaRESTAPIClient, node_ip: IPAddress, injection: str) -> InjectionHandler:
async def inject_error(api: ScyllaRESTAPIClient, node_ip: IPAddress, injection: str, parameters: dict[str, Any] = {}) -> InjectionHandler:
"""Attempts to inject an error. Works only in specific build modes: debug,dev,sanitize.
It will trigger a test to be skipped if attempting to enable an injection has no effect.
This is a context manager for enabling and disabling when done, therefore it can't be
used for one shot.
"""
await api.enable_injection(node_ip, injection, False)
await api.enable_injection(node_ip, injection, False, parameters)
enabled = await api.get_enabled_injections(node_ip)
logging.info(f"Error injections enabled on {node_ip}: {enabled}")
if not enabled:
@@ -255,12 +255,12 @@ async def inject_error(api: ScyllaRESTAPIClient, node_ip: IPAddress, injection:
await api.disable_injection(node_ip, injection)
async def inject_error_one_shot(api: ScyllaRESTAPIClient, node_ip: IPAddress, injection: str) -> InjectionHandler:
async def inject_error_one_shot(api: ScyllaRESTAPIClient, node_ip: IPAddress, injection: str, parameters: dict[str, Any] = {}) -> InjectionHandler:
"""Attempts to inject an error. Works only in specific build modes: debug,dev,sanitize.
It will trigger a test to be skipped if attempting to enable an injection has no effect.
This is a one-shot injection enable.
"""
await api.enable_injection(node_ip, injection, True)
await api.enable_injection(node_ip, injection, True, parameters)
enabled = await api.get_enabled_injections(node_ip)
logging.info(f"Error injections enabled on {node_ip}: {enabled}")
if not enabled:

67
utils/error_injection.hh
View File

@@ -24,6 +24,7 @@
#include <type_traits>
#include <concepts>
#include <optional>
#include <unordered_map>
#include <boost/range/adaptor/map.hpp>
#include <boost/range/adaptor/filtered.hpp>
@@ -40,6 +41,8 @@ public:
extern logging::logger errinj_logger;
using error_injection_parameters = std::unordered_map<sstring, sstring>;
/**
* Error injection class can be used to create and manage code injections
* which trigger an error or a custom action in debug mode.
@@ -113,9 +116,13 @@ class error_injection {
* It is shared between the injection_data. It is created once when enabling an injection
* on a given shard, and all injection_handlers, that are created separately for each firing of this injection.
*/
struct received_messages_counter {
size_t counter{0};
condition_variable cv;
struct injection_shared_data {
size_t received_message_count{0};
condition_variable received_message_cv;
error_injection_parameters parameters;
explicit injection_shared_data(error_injection_parameters parameters)
: parameters(std::move(parameters)) {}
};
class injection_data;
@@ -126,21 +133,23 @@ public:
* all of them will have separate handlers.
*/
class injection_handler: public bi::list_base_hook<bi::link_mode<bi::auto_unlink>> {
lw_shared_ptr<received_messages_counter> _received_counter;
lw_shared_ptr<injection_shared_data> _shared_data;
size_t _read_messages_counter{0};
explicit injection_handler(lw_shared_ptr<received_messages_counter> received_counter)
: _received_counter(std::move(received_counter)) {}
explicit injection_handler(lw_shared_ptr<injection_shared_data> shared_data)
: _shared_data(std::move(shared_data)) {}
public:
template <typename Clock, typename Duration>
future<> wait_for_message(std::chrono::time_point<Clock, Duration> timeout) {
if (!_received_counter) {
on_internal_error(errinj_logger, "received_messages_counter is not initialized");
if (!_shared_data) {
on_internal_error(errinj_logger, "injection_shared_data is not initialized");
}
try {
co_await _received_counter->cv.wait(timeout, [&] { return _read_messages_counter < _received_counter->counter; });
co_await _shared_data->received_message_cv.wait(timeout, [&] {
return _read_messages_counter < _shared_data->received_message_count;
});
}
catch (const std::exception& e) {
on_internal_error(errinj_logger, "Error injection wait_for_message timeout: " + std::string(e.what()));
@@ -148,6 +157,18 @@ public:
++_read_messages_counter;
}
std::optional<std::string_view> get(std::string_view key) {
if (!_shared_data) {
on_internal_error(errinj_logger, "injection_shared_data is not initialized");
}
auto it = _shared_data->parameters.find(std::string(key));
if (it == _shared_data->parameters.end()) {
return std::nullopt;
}
return it->second;
}
friend class error_injection;
};
@@ -163,18 +184,18 @@ private:
*/
struct injection_data {
bool one_shot;
lw_shared_ptr<received_messages_counter> received_counter;
lw_shared_ptr<injection_shared_data> shared_data;
bi::list<injection_handler, bi::constant_time_size<false>> handlers;
explicit injection_data(bool one_shot)
explicit injection_data(bool one_shot, error_injection_parameters parameters)
: one_shot(one_shot)
, received_counter(make_lw_shared<received_messages_counter>()) {}
, shared_data(make_lw_shared<injection_shared_data>(std::move(parameters))) {}
void receive_message() {
assert(received_counter);
assert(shared_data);
++received_counter->counter;
received_counter->cv.broadcast();
++shared_data->received_message_count;
shared_data->received_message_cv.broadcast();
}
bool is_one_shot() const {
@@ -235,8 +256,8 @@ public:
return true;
}
void enable(const std::string_view& injection_name, bool one_shot = false) {
_enabled.emplace(injection_name, injection_data{one_shot});
void enable(const std::string_view& injection_name, bool one_shot = false, error_injection_parameters parameters = {}) {
_enabled.emplace(injection_name, injection_data{one_shot, std::move(parameters)});
errinj_logger.debug("Enabling injection {} \"{}\"",
one_shot? "one-shot ": "", injection_name);
}
@@ -364,7 +385,7 @@ public:
}
errinj_logger.debug("Triggering injection \"{}\" with injection handler", name);
injection_handler handler(data->received_counter);
injection_handler handler(data->shared_data);
data->handlers.push_back(handler);
auto disable_one_shot = defer([this, one_shot, name = sstring(name)] {
@@ -376,10 +397,10 @@ public:
co_await func(handler);
}
future<> enable_on_all(const std::string_view& injection_name, bool one_shot = false) {
return smp::invoke_on_all([injection_name = sstring(injection_name), one_shot] {
future<> enable_on_all(const std::string_view& injection_name, bool one_shot = false, error_injection_parameters parameters = {}) {
return smp::invoke_on_all([injection_name = sstring(injection_name), one_shot, parameters = std::move(parameters)] {
auto& errinj = _local;
errinj.enable(injection_name, one_shot);
errinj.enable(injection_name, one_shot, parameters);
});
}
@@ -436,7 +457,7 @@ public:
}
[[gnu::always_inline]]
void enable(const std::string_view& injection_name, const bool one_shot = false) {}
void enable(const std::string_view& injection_name, const bool one_shot = false, error_injection_parameters parameters = {}) {}
[[gnu::always_inline]]
void disable(const std::string_view& injection_name) {}
@@ -495,7 +516,7 @@ public:
}
[[gnu::always_inline]]
static future<> enable_on_all(const std::string_view& injection_name, const bool one_shot = false) {
static future<> enable_on_all(const std::string_view& injection_name, const bool one_shot = false, const error_injection_parameters& parameters = {}) {
return make_ready_future<>();
}