From c35ed794de7e98b61e3e857f7355ac2c59c8155a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Botond=20D=C3=A9nes?= Date: Fri, 8 Dec 2023 07:05:42 -0500 Subject: [PATCH] tools/scylla-nodetool: implement the removenode command --- test/nodetool/test_nodeops.py | 58 +++++++++++++++++++++++++++++++++++ tools/scylla-nodetool.cc | 43 ++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/test/nodetool/test_nodeops.py b/test/nodetool/test_nodeops.py index e0c2c69c03..8788e515ae 100644 --- a/test/nodetool/test_nodeops.py +++ b/test/nodetool/test_nodeops.py @@ -5,6 +5,7 @@ # from rest_api_mock import expected_request +import utils def test_decommission(nodetool): @@ -20,3 +21,60 @@ def test_rebuild(nodetool): def test_rebuild_with_dc(nodetool): nodetool("rebuild", "DC1", expected_requests=[ expected_request("POST", "/storage_service/rebuild", params={"source_dc": "DC1"})]) + + +def test_removenode(nodetool): + nodetool("removenode", "675ed9f4-6564-6dbd-can8-43fddce952gy", expected_requests=[ + expected_request("POST", "/storage_service/remove_node", + params={"host_id": "675ed9f4-6564-6dbd-can8-43fddce952gy"})]) + + +def test_removenode_ignore_nodes_one_node(nodetool): + nodetool("removenode", + "675ed9f4-6564-6dbd-can8-43fddce952gy", + "--ignore-dead-nodes", + "88eed9f4-6564-6dbd-can8-43fddce952gy", + expected_requests=[ + expected_request("POST", "/storage_service/remove_node", params={ + "host_id": "675ed9f4-6564-6dbd-can8-43fddce952gy", + "ignore_nodes": "88eed9f4-6564-6dbd-can8-43fddce952gy"})]) + + +def test_removenode_ignore_nodes_two_nodes(nodetool): + nodetool("removenode", + "675ed9f4-6564-6dbd-can8-43fddce952gy", + "--ignore-dead-nodes", + "88eed9f4-6564-6dbd-can8-43fddce952gy,99eed9f4-6564-6dbd-can8-43fddce952gy", + expected_requests=[ + expected_request("POST", "/storage_service/remove_node", params={ + "host_id": "675ed9f4-6564-6dbd-can8-43fddce952gy", + "ignore_nodes": "88eed9f4-6564-6dbd-can8-43fddce952gy,99eed9f4-6564-6dbd-can8-43fddce952gy"})]) + + +def test_removenode_status(nodetool): + res = nodetool("removenode", "status", expected_requests=[ + expected_request("GET", "/storage_service/removal_status", response="SOME STATUS")]) + assert res == "RemovalStatus: SOME STATUS\n" + + +def test_removenode_force(nodetool): + res = nodetool("removenode", "force", expected_requests=[ + expected_request("GET", "/storage_service/removal_status", response="SOME STATUS"), + expected_request("POST", "/storage_service/force_remove_completion")]) + assert res == "RemovalStatus: SOME STATUS\n" + + +def test_removenode_status_with_ignore_dead_nodes(nodetool, scylla_only): + utils.check_nodetool_fails_with( + nodetool, + ("removenode", "status", "--ignore-dead-nodes", "675ed9f4-6564-6dbd-can8-43fddce952gy"), + {"expected_requests": []}, + ["error processing arguments: cannot use --ignore-dead-nodes with status or force"]) + + +def test_removenode_force_with_ignore_dead_nodes(nodetool, scylla_only): + utils.check_nodetool_fails_with( + nodetool, + ("removenode", "force", "--ignore-dead-nodes", "675ed9f4-6564-6dbd-can8-43fddce952gy"), + {"expected_requests": []}, + ["error processing arguments: cannot use --ignore-dead-nodes with status or force"]) diff --git a/tools/scylla-nodetool.cc b/tools/scylla-nodetool.cc index 0ce2a95440..6eb5cde373 100644 --- a/tools/scylla-nodetool.cc +++ b/tools/scylla-nodetool.cc @@ -548,6 +548,28 @@ void rebuild_operation(scylla_rest_client& client, const bpo::variables_map& vm) client.post("/storage_service/rebuild", std::move(params)); } +void removenode_operation(scylla_rest_client& client, const bpo::variables_map& vm) { + if (!vm.count("remove-operation")) { + throw std::invalid_argument("required parameters are missing: remove-operation, pass one of (status, force or a host id)"); + } + const auto op = vm["remove-operation"].as(); + if (op == "status" || op == "force") { + if (vm.count("ignore-dead-nodes")) { + throw std::invalid_argument("cannot use --ignore-dead-nodes with status or force"); + } + fmt::print(std::cout, "RemovalStatus: {}\n", rjson::to_string_view(client.get("/storage_service/removal_status"))); + if (op == "force") { + client.post("/storage_service/force_remove_completion"); + } + } else { + std::unordered_map params{{"host_id", op}}; + if (vm.count("ignore-dead-nodes")) { + params["ignore_nodes"] = vm["ignore-dead-nodes"].as(); + } + client.post("/storage_service/remove_node", std::move(params)); + } +} + void settraceprobability_operation(scylla_rest_client& client, const bpo::variables_map& vm) { if (!vm.count("trace_probability")) { throw std::invalid_argument("required parameters are missing: trace_probability"); @@ -984,6 +1006,27 @@ Fore more information, see: https://opensource.docs.scylladb.com/stable/operatin }, rebuild_operation }, + { + { + "removenode", + "Remove a node from the cluster when the status of the node is Down Normal (DN) and all attempts to restore the node have failed", +R"( +Provide the Host ID of the node to specify which node you want to remove. + +Important: use this command *only* on nodes that are not reachable by other nodes +by any means! + +Fore more information, see: https://opensource.docs.scylladb.com/stable/operating-scylla/nodetool-commands/removenode.html +)", + { + typed_option("ignore-dead-nodes", "Comma-separated list of dead nodes to ignore during removenode"), + }, + { + typed_option("remove-operation", "status|force|$HOST_ID - show status of current node removal, force completion of pending removal, or remove provided ID", 1), + }, + }, + removenode_operation + }, { { "settraceprobability",