bahman/scylla
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge 'Nodetool additional commands 3/N' from Botond Dénes

Browse Source 
This PR implements the following new nodetool commands:
* cleanup
* clearsnapshots
* listsnapshots

All commands come with tests and all tests pass with both the new and the current nodetool implementations.

Refs: https://github.com/scylladb/scylladb/issues/15588

Closes scylladb/scylladb#15843

* github.com:scylladb/scylladb:
  tools/scylla-nodetool: implement the listsnapshots command
  tools/scylla-nodetool: implement clearsnapshot command
  tools/scylla-nodetool: implement the cleanup command
  test/nodetool: rest_api_mock: add more options for multiple requests
  tools/scylla-nodetool: log responses with trace level
This commit is contained in:
Avi Kivity
2023-10-30 21:53:36 +02:00
parent 5a7d15a666 16ce212c31
commit 949e9f1205
5 changed files with 375 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
test/nodetool/rest_api_mock.py
View File

@@ -20,7 +20,11 @@ logger = logging.getLogger(__name__)
class expected_request:
def __init__(self, method: str, path: str, params: dict = {}, multiple: bool = False,
ANY = -1 # allow for any number of requests (including no requests at all), similar to the `*` quantity in regexp
ONE = 0 # exactly one request is allowed
MULTIPLE = 1 # one or more request is allowed
def __init__(self, method: str, path: str, params: dict = {}, multiple: int = ONE,
response: Dict[str, Any] = None, response_status: int = 200):
self.method = method
self.path = path
@@ -52,7 +56,7 @@ def _make_expected_request(req_json):
req_json["method"],
req_json["path"],
params=req_json.get("params", dict()),
multiple=req_json.get("multiple", False),
multiple=req_json.get("multiple", expected_request.ONE),
response=req_json.get("response"),
response_status=req_json.get("response_status", 200))
@@ -130,19 +134,24 @@ class rest_server(aiohttp.abc.AbstractRouter):
return aiohttp.web.Response(status=500, text="Expected no requests, got {this_req}")
expected_req = self.expected_requests[0]
if this_req != expected_req:
if expected_req.multiple and expected_req.hit > 0 and \
len(self.expected_requests) > 1 and self.expected_requests[1] == this_req:
while this_req != expected_req:
if expected_req.multiple == expected_request.ANY or (
expected_req.multiple >= expected_request.MULTIPLE and expected_req.hit >= expected_req.multiple):
logger.info(f"popping multi request {expected_req}")
del self.expected_requests[0]
expected_req = self.expected_requests[0]
else:
logger.error(f"unexpected request, expected {expected_req}, got {this_req}")
return aiohttp.web.Response(status=500, text="Expected {expected_req}, got {this_req}")
if not expected_req.multiple:
if len(self.expected_requests) > 0:
expected_req = self.expected_requests[0]
continue
logger.error(f"unexpected request, expected {expected_req}, got {this_req}")
return aiohttp.web.Response(status=500, text="Expected {expected_req}, got {this_req}")
if expected_req.multiple == expected_request.ONE:
del self.expected_requests[0]
expected_req.hit += 1
else:
expected_req.hit += 1
if expected_req.response is None:
logger.info(f"expected_request: {expected_req}, no response")

73
test/nodetool/test_cleanup.py Normal file
View File

@@ -0,0 +1,73 @@
#
# Copyright 2023-present ScyllaDB
#
# SPDX-License-Identifier: AGPL-3.0-or-later
#
from rest_api_mock import expected_request
import utils
def test_cleanup(nodetool):
nodetool("cleanup", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", params={"type": "non_local_strategy"},
response=["ks1", "ks2"]),
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.ANY,
response=["ks1", "ks2", "system"]),
expected_request("POST", "/storage_service/keyspace_cleanup/ks1", response=0),
expected_request("POST", "/storage_service/keyspace_cleanup/ks2", response=0),
])
def test_cleanup_keyspace(nodetool):
nodetool("cleanup", "ks1", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["ks1", "ks2", "system"]),
expected_request("POST", "/storage_service/keyspace_cleanup/ks1", response=0),
])
def test_cleanup_table(nodetool):
nodetool("cleanup", "ks1", "tbl1", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["ks1", "ks2", "system"]),
expected_request("POST", "/storage_service/keyspace_cleanup/ks1", params={"cf": "tbl1"}, response=0),
])
def test_cleanup_tables(nodetool):
nodetool("cleanup", "ks1", "tbl1", "tbl2", "tbl3", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["ks1", "ks2", "system"]),
expected_request("POST", "/storage_service/keyspace_cleanup/ks1", params={"cf": "tbl1,tbl2,tbl3"}, response=0),
])
def test_cleanup_nonexistent_keyspace(nodetool):
utils.check_nodetool_fails_with(
nodetool,
("cleanup", "non_existent_ks"),
{"expected_requests": [
expected_request("GET", "/storage_service/keyspaces", response=["ks1", "ks2", "system"])]},
["nodetool: Keyspace [non_existent_ks] does not exist.",
"error processing arguments: keyspace non_existent_ks does not exist"])
def test_cleanup_jobs_arg(nodetool):
nodetool("cleanup", "ks1", "-j", "0", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["ks1", "ks2", "system"]),
expected_request("POST", "/storage_service/keyspace_cleanup/ks1", response=0),
])
nodetool("cleanup", "ks1", "--jobs", "2", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["ks1", "ks2", "system"]),
expected_request("POST", "/storage_service/keyspace_cleanup/ks1", response=0),
])
nodetool("cleanup", "ks1", "--jobs=1", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["ks1", "ks2", "system"]),
expected_request("POST", "/storage_service/keyspace_cleanup/ks1", response=0),
])

21
test/nodetool/test_compact.py
View File

@@ -10,14 +10,16 @@ import utils
def test_all_keyspaces(nodetool):
nodetool("compact", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=True, response=["system", "system_schema"]),
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["system", "system_schema"]),
expected_request("POST", "/storage_service/keyspace_compaction/system"),
expected_request("POST", "/storage_service/keyspace_compaction/system_schema")])
def test_keyspace(nodetool):
nodetool("compact", "system_schema", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=True, response=["system", "system_schema"]),
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["system", "system_schema"]),
expected_request("POST", "/storage_service/keyspace_compaction/system_schema")])
@@ -26,7 +28,8 @@ def test_nonexistent_keyspace(nodetool):
nodetool,
("compact", "non_existent_ks"),
{"expected_requests": [
expected_request("GET", "/storage_service/keyspaces", multiple=True, response=["system"]),
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["system"]),
expected_request("POST", "/storage_service/keyspace_compaction/non_existent_ks")]},
["nodetool: Keyspace [non_existent_ks] does not exist.",
"error processing arguments: keyspace non_existent_ks does not exist"])
@@ -34,11 +37,13 @@ def test_nonexistent_keyspace(nodetool):
def test_table(nodetool):
nodetool("compact", "system_schema", "columns", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=True, response=["system", "system_schema"]),
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["system", "system_schema"]),
expected_request("POST", "/storage_service/keyspace_compaction/system_schema", params={"cf": "columns"})])
nodetool("compact", "system_schema", "columns", "computed_columns", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=True, response=["system", "system_schema"]),
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["system", "system_schema"]),
expected_request("POST",
"/storage_service/keyspace_compaction/system_schema",
params={"cf": "columns,computed_columns"})])
@@ -46,7 +51,8 @@ def test_table(nodetool):
def test_split_output_compatibility_argument(nodetool):
dummy_request = [
expected_request("GET", "/storage_service/keyspaces", multiple=True, response=["system", "system_schema"]),
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["system", "system_schema"]),
expected_request("POST", "/storage_service/keyspace_compaction/system_schema")]
nodetool("compact", "system_schema", "-s", expected_requests=dummy_request)
@@ -55,7 +61,8 @@ def test_split_output_compatibility_argument(nodetool):
def test_token_range_compatibility_argument(nodetool):
dummy_request = [
expected_request("GET", "/storage_service/keyspaces", multiple=True, response=["system", "system_schema"]),
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.MULTIPLE,
response=["system", "system_schema"]),
expected_request("POST", "/storage_service/keyspace_compaction/system_schema")]
nodetool("compact", "system_schema", "-st", "0", "-et", "1000", expected_requests=dummy_request)

101
test/nodetool/test_snapshot.py Normal file
View File

@@ -0,0 +1,101 @@
#
# Copyright 2023-present ScyllaDB
#
# SPDX-License-Identifier: AGPL-3.0-or-later
#
from rest_api_mock import expected_request
import utils
def test_clearnapshot(nodetool):
nodetool("clearsnapshot", expected_requests=[
expected_request("DELETE", "/storage_service/snapshots")
])
def test_clearnapshot_keyspace(nodetool):
nodetool("clearsnapshot", "ks1", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.ANY,
response=["ks1", "ks2"]),
expected_request("DELETE", "/storage_service/snapshots", params={"kn": "ks1"})
])
def test_clearnapshot_keyspaces(nodetool):
nodetool("clearsnapshot", "ks1", "ks2", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.ANY,
response=["ks1", "ks2"]),
expected_request("DELETE", "/storage_service/snapshots", params={"kn": "ks1,ks2"})
])
def test_clearnapshot_nonexistent_keyspaces(nodetool, scylla_only):
utils.check_nodetool_fails_with(
nodetool,
("clearsnapshot", "non_existent_ks"),
{"expected_requests": [
expected_request("GET", "/storage_service/keyspaces", response=["ks1", "ks2"])]},
["error processing arguments: keyspace non_existent_ks does not exist"])
def test_clearnapshot_tag(nodetool):
nodetool("clearsnapshot", "-t", "snapshot_name", expected_requests=[
expected_request("DELETE", "/storage_service/snapshots", params={"tag": "snapshot_name"})
])
def test_clearnapshot_tag_and_keyspace(nodetool):
nodetool("clearsnapshot", "-t", "snapshot_name", "ks1", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.ANY,
response=["ks1", "ks2"]),
expected_request("DELETE", "/storage_service/snapshots", params={"kn": "ks1", "tag": "snapshot_name"})
])
def test_clearnapshot_tag_and_keyspaces(nodetool):
nodetool("clearsnapshot", "-t", "snapshot_name", "ks1", "ks2", expected_requests=[
expected_request("GET", "/storage_service/keyspaces", multiple=expected_request.ANY,
response=["ks1", "ks2"]),
expected_request("DELETE", "/storage_service/snapshots", params={"kn": "ks1,ks2", "tag": "snapshot_name"})
])
def test_listsnapshots(nodetool, request):
res = nodetool("listsnapshots", expected_requests=[
expected_request("GET", "/storage_service/snapshots", response=[
{"key": "1698236289867", "value": [{"ks": "ks1", "cf": "tbl1", "total": 45056, "live": 0},
{"ks": "ks1", "cf": "tbl2", "total": 40956, "live": 0}]},
{"key": "1698236070745", "value": [{"ks": "ks1", "cf": "tbl1", "total": 35056, "live": 0},
{"ks": "ks1", "cf": "tbl2", "total": 20956, "live": 0}]},
]),
expected_request("GET", "/storage_service/snapshots/size/true", response=945235),
])
cassandra_expected_output =\
"""Snapshot Details:
Snapshot name Keyspace name Column family name True size Size on disk
1698236289867 ks1 tbl1 0 bytes 44 KB
1698236289867 ks1 tbl2 0 bytes 40 KB
1698236070745 ks1 tbl1 0 bytes 34.23 KB
1698236070745 ks1 tbl2 0 bytes 20.46 KB
Total TrueDiskSpaceUsed: 923.08 KiB
"""
scylla_expected_output =\
"""Snapshot Details:
Snapshot name Keyspace name Column family name True size Size on disk
1698236289867 ks1 tbl1 0 B 44 KiB
1698236289867 ks1 tbl2 0 B 40 KiB
1698236070745 ks1 tbl1 0 B 34 KiB
1698236070745 ks1 tbl2 0 B 20 KiB
Total TrueDiskSpaceUsed: 923 KiB
"""
if request.config.getoption("nodetool") == "scylla":
assert res == scylla_expected_output
else:
assert res == cassandra_expected_output

172
tools/scylla-nodetool.cc
View File

@@ -20,6 +20,7 @@
#include "log.hh"
#include "tools/utils.hh"
#include "utils/http.hh"
#include "utils/human_readable.hh"
#include "utils/rjson.hh"
#include "utils/UUID.hh"
@@ -58,8 +59,10 @@ class scylla_rest_client {
}
if (res.empty()) {
nlog.trace("Got empty response");
return rjson::null_value();
} else {
nlog.trace("Got response:\n{}", res);
return rjson::parse(res);
}
}
@@ -90,20 +93,76 @@ public:
}
};
std::vector<sstring> get_keyspaces(scylla_rest_client& client, std::optional<sstring> type = {}) {
std::unordered_map<sstring, sstring> params;
if (type) {
params["type"] = *type;
}
auto keyspaces_json = client.get("/storage_service/keyspaces", std::move(params));
std::vector<sstring> keyspaces;
for (const auto& keyspace_json : keyspaces_json.GetArray()) {
keyspaces.emplace_back(rjson::to_string_view(keyspace_json));
}
return keyspaces;
}
using operation_func = void(*)(scylla_rest_client&, const bpo::variables_map&);
std::map<operation, operation_func> get_operations_with_func();
void cleanup_operation(scylla_rest_client& client, const bpo::variables_map& vm) {
if (vm.count("cleanup_arg")) {
const auto all_keyspaces = get_keyspaces(client);
auto args = vm["cleanup_arg"].as<std::vector<sstring>>();
std::unordered_map<sstring, sstring> params;
const auto keyspace = args[0];
if (std::ranges::find(all_keyspaces, keyspace) == all_keyspaces.end()) {
throw std::invalid_argument(fmt::format("keyspace {} does not exist", keyspace));
}
if (args.size() > 1) {
params["cf"] = fmt::to_string(fmt::join(args.begin() + 1, args.end(), ","));
}
client.post(format("/storage_service/keyspace_cleanup/{}", keyspace), std::move(params));
} else {
for (const auto& keyspace : get_keyspaces(client, "non_local_strategy")) {
client.post(format("/storage_service/keyspace_cleanup/{}", keyspace));
}
}
}
void clearsnapshot_operation(scylla_rest_client& client, const bpo::variables_map& vm) {
std::unordered_map<sstring, sstring> params;
if (vm.count("keyspaces")) {
std::vector<sstring> keyspaces;
const auto all_keyspaces = get_keyspaces(client);
for (const auto& keyspace : vm["keyspaces"].as<std::vector<sstring>>()) {
if (std::ranges::find(all_keyspaces, keyspace) == all_keyspaces.end()) {
throw std::invalid_argument(fmt::format("keyspace {} does not exist", keyspace));
}
keyspaces.push_back(keyspace);
}
if (!keyspaces.empty()) {
params["kn"] = fmt::to_string(fmt::join(keyspaces.begin(), keyspaces.end(), ","));
}
}
if (vm.count("tag")) {
params["tag"] = vm["tag"].as<sstring>();
}
client.del("/storage_service/snapshots", std::move(params));
}
void compact_operation(scylla_rest_client& client, const bpo::variables_map& vm) {
if (vm.count("user-defined")) {
throw std::invalid_argument("--user-defined flag is unsupported");
}
auto keyspaces_json = client.get("/storage_service/keyspaces", {});
std::vector<sstring> all_keyspaces;
for (const auto& keyspace_json : keyspaces_json.GetArray()) {
all_keyspaces.emplace_back(rjson::to_string_view(keyspace_json));
}
const auto all_keyspaces = get_keyspaces(client);
if (vm.count("compaction_arg")) {
auto args = vm["compaction_arg"].as<std::vector<sstring>>();
@@ -280,6 +339,54 @@ void gettraceprobability_operation(scylla_rest_client& client, const bpo::variab
fmt::print(std::cout, "Current trace probability: {}\n", res.GetDouble());
}
void listsnapshots_operation(scylla_rest_client& client, const bpo::variables_map& vm) {
const auto snapshots = client.get("/storage_service/snapshots");
const auto true_size = client.get("/storage_service/snapshots/size/true").GetInt64();
std::array<std::string, 5> header_row{"Snapshot name", "Keyspace name", "Column family name", "True size", "Size on disk"};
std::array<size_t, 5> max_column_length{};
for (size_t c = 0; c < header_row.size(); ++c) {
max_column_length[c] = header_row[c].size();
}
auto format_hr_size = [] (const utils::human_readable_value hrv) {
if (!hrv.suffix || hrv.suffix == 'B') {
return fmt::format("{} B ", hrv.value);
}
return fmt::format("{} {}iB", hrv.value, hrv.suffix);
};
std::vector<std::array<std::string, 5>> rows;
for (const auto& snapshot_by_name : snapshots.GetArray()) {
const auto snapshot_name = std::string(rjson::to_string_view(snapshot_by_name.GetObject()["key"]));
for (const auto& snapshot : snapshot_by_name.GetObject()["value"].GetArray()) {
rows.push_back({
snapshot_name,
std::string(rjson::to_string_view(snapshot["ks"])),
std::string(rjson::to_string_view(snapshot["cf"])),
format_hr_size(utils::to_hr_size(snapshot["live"].GetInt64())),
format_hr_size(utils::to_hr_size(snapshot["total"].GetInt64()))});
for (size_t c = 0; c < rows.back().size(); ++c) {
max_column_length[c] = std::max(max_column_length[c], rows.back()[c].size());
}
}
}
const auto header_row_format = fmt::format("{{:<{}}} {{:<{}}} {{:<{}}} {{:<{}}} {{:<{}}}\n", max_column_length[0],
max_column_length[1], max_column_length[2], max_column_length[3], max_column_length[4]);
const auto regular_row_format = fmt::format("{{:<{}}} {{:<{}}} {{:<{}}} {{:>{}}} {{:>{}}}\n", max_column_length[0],
max_column_length[1], max_column_length[2], max_column_length[3], max_column_length[4]);
fmt::print(std::cout, "Snapshot Details:\n");
fmt::print(std::cout, fmt::runtime(header_row_format.c_str()), header_row[0], header_row[1], header_row[2], header_row[3], header_row[4]);
for (const auto& r : rows) {
fmt::print(std::cout, fmt::runtime(regular_row_format.c_str()), r[0], r[1], r[2], r[3], r[4]);
}
fmt::print(std::cout, "\nTotal TrueDiskSpaceUsed: {}\n\n", format_hr_size(utils::to_hr_size(true_size)));
}
void help_operation(const tool_app_template::config& cfg, const bpo::variables_map& vm) {
if (vm.count("command")) {
const auto command = vm["command"].as<sstring>();
@@ -416,6 +523,47 @@ const std::map<std::string_view, std::string_view> option_substitutions{
std::map<operation, operation_func> get_operations_with_func() {
const static std::map<operation, operation_func> operations_with_func {
{
{
"cleanup",
"Triggers removal of data that the node no longer owns",
R"(
You should run nodetool cleanup whenever you scale-out (expand) your cluster, and
new nodes are added to the same DC. The scale out process causes the token ring
to get re-distributed. As a result, some of the nodes will have replicas for
tokens that they are no longer responsible for (taking up disk space). This data
continues to consume diskspace until you run nodetool cleanup. The cleanup
operation deletes these replicas and frees up disk space.
Fore more information, see: https://opensource.docs.scylladb.com/stable/operating-scylla/nodetool-commands/cleanup.html
)",
{
typed_option<int64_t>("jobs,j", "The number of compaction jobs to be used for the cleanup (unused)"),
},
{
typed_option<std::vector<sstring>>("cleanup_arg", "[<keyspace> <tables>...]", -1),
}
},
cleanup_operation
},
{
{
"clearsnapshot",
"Remove snapshots",
R"(
By default all snapshots are removed for all keyspaces.
Fore more information, see: https://opensource.docs.scylladb.com/stable/operating-scylla/nodetool-commands/clearsnapshot.html
)",
{
typed_option<sstring>("tag,t", "The snapshot to remove"),
},
{
typed_option<std::vector<sstring>>("keyspaces", "[<keyspaces>...]", -1),
}
},
clearsnapshot_operation
},
{
{
"compact",
@@ -545,6 +693,20 @@ Fore more information, see: https://opensource.docs.scylladb.com/stable/operatin
},
[] (scylla_rest_client&, const bpo::variables_map&) {}
},
{
{
"listsnapshots",
"Lists all the snapshots along with the size on disk and true size",
R"(
Dropped tables (column family) will not be part of the listsnapshots.
Fore more information, see: https://opensource.docs.scylladb.com/stable/operating-scylla/nodetool-commands/listsnapshots.html
)",
{ },
{ },
},
listsnapshots_operation
},
{
{
"settraceprobability",