From 7cbe5e3af8fc2bfbede9c73bfd9a5475bdd519cd Mon Sep 17 00:00:00 2001 From: Piotr Dulikowski Date: Tue, 13 Jun 2023 21:51:23 +0200 Subject: [PATCH] rpc: add new join handshake verbs The `join_node_request` and `join_node_response` RPCs are added: - `join_node_request` is sent from the joining node to any node in the cluster. It contains some initial parameters that will be verified by the receiving node, or the topology coordinator - notably, it contains a list of cluster features supported by the joining node. - `join_node_response` is sent from the topology coordinator to the joining node to tell it about the the outcome of the verification. --- configure.py | 1 + idl/join_node.idl.hh | 56 ++++++++++++++++++++++++++++++++ message/messaging_service.cc | 5 +++ message/messaging_service.hh | 4 ++- service/raft/join_node.hh | 62 ++++++++++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 idl/join_node.idl.hh create mode 100644 service/raft/join_node.hh diff --git a/configure.py b/configure.py index 8fcca5cd3f..80ede7df7b 100755 --- a/configure.py +++ b/configure.py @@ -1318,6 +1318,7 @@ idls = ['idl/gossip_digest.idl.hh', 'idl/position_in_partition.idl.hh', 'idl/experimental/broadcast_tables_lang.idl.hh', 'idl/storage_service.idl.hh', + 'idl/join_node.idl.hh', 'idl/utils.idl.hh', ] diff --git a/idl/join_node.idl.hh b/idl/join_node.idl.hh new file mode 100644 index 0000000000..cd3db5aee8 --- /dev/null +++ b/idl/join_node.idl.hh @@ -0,0 +1,56 @@ +/* + * Copyright 2023-present ScyllaDB + */ + +/* + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace service { + +struct join_node_request_params { + raft::server_id host_id; + std::optional replaced_id; + std::vector ignore_nodes; + sstring cluster_name; + sstring snitch_name; + sstring datacenter; + sstring rack; + sstring release_version; + uint32_t num_tokens; + uint32_t shard_count; + uint32_t ignore_msb; + std::vector supported_features; +}; + +struct join_node_request_result { + struct ok {}; + struct rejected { + sstring reason; + }; + + std::variant< + service::join_node_request_result::ok, + service::join_node_request_result::rejected + > result; +}; + +struct join_node_response_params { + struct accepted {}; + + struct rejected { + sstring reason; + }; + + std::variant< + service::join_node_response_params::accepted, + service::join_node_response_params::rejected + > response; +}; + +struct join_node_response_result {}; + +verb join_node_request (raft::server_id dst_id, service::join_node_request_params) -> service::join_node_request_result; +verb join_node_response (raft::server_id dst_id, service::join_node_response_params) -> service::join_node_response_result; + +} diff --git a/message/messaging_service.cc b/message/messaging_service.cc index 00e56dad29..6c0f806bbf 100644 --- a/message/messaging_service.cc +++ b/message/messaging_service.cc @@ -48,6 +48,7 @@ #include "full_position.hh" #include "db/per_partition_rate_limit_info.hh" #include "service/topology_state_machine.hh" +#include "service/raft/join_node.hh" #include "idl/consistency_level.dist.hh" #include "idl/tracing.dist.hh" #include "idl/result.dist.hh" @@ -77,6 +78,7 @@ #include "idl/per_partition_rate_limit_info.dist.hh" #include "idl/storage_proxy.dist.hh" #include "idl/storage_service.dist.hh" +#include "idl/join_node.dist.hh" #include "message/rpc_protocol_impl.hh" #include "idl/consistency_level.dist.impl.hh" #include "idl/tracing.dist.impl.hh" @@ -118,6 +120,7 @@ #include "idl/forward_request.dist.hh" #include "idl/forward_request.dist.impl.hh" #include "idl/storage_service.dist.impl.hh" +#include "idl/join_node.dist.impl.hh" namespace netw { @@ -566,6 +569,8 @@ static constexpr unsigned do_get_rpc_client_idx(messaging_verb verb) { case messaging_verb::GROUP0_MODIFY_CONFIG: case messaging_verb::GET_GROUP0_UPGRADE_STATE: case messaging_verb::RAFT_TOPOLOGY_CMD: + case messaging_verb::JOIN_NODE_REQUEST: + case messaging_verb::JOIN_NODE_RESPONSE: // See comment above `TOPOLOGY_INDEPENDENT_IDX`. // DO NOT put any 'hot' (e.g. data path) verbs in this group, // only verbs which are 'rare' and 'cheap'. diff --git a/message/messaging_service.hh b/message/messaging_service.hh index a5b88f13a6..71e1bc0866 100644 --- a/message/messaging_service.hh +++ b/message/messaging_service.hh @@ -184,7 +184,9 @@ enum class messaging_verb : int32_t { RAFT_PULL_TOPOLOGY_SNAPSHOT = 65, TABLET_STREAM_DATA = 66, TABLET_CLEANUP = 67, - LAST = 68, + JOIN_NODE_REQUEST = 68, + JOIN_NODE_RESPONSE = 69, + LAST = 70, }; } // namespace netw diff --git a/service/raft/join_node.hh b/service/raft/join_node.hh new file mode 100644 index 0000000000..fe7d22f7f0 --- /dev/null +++ b/service/raft/join_node.hh @@ -0,0 +1,62 @@ +/* + * Copyright 2023-present ScyllaDB + */ + +/* + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +#pragma once + +#include +#include +#include "raft/raft.hh" + +namespace service { + +struct join_node_request_params { + raft::server_id host_id; + std::optional replaced_id; + std::vector ignore_nodes; + sstring cluster_name; + sstring snitch_name; + sstring datacenter; + sstring rack; + sstring release_version; + uint32_t num_tokens; + uint32_t shard_count; + uint32_t ignore_msb; + std::vector supported_features; +}; + +struct join_node_request_result { + // Request was successfully placed and will be processed + // by the topology coordinator. + struct ok {}; + + // The request was immediately rejected, most likely due to some + // parameters being incorrect or incompatible with the cluster. + struct rejected { + sstring reason; + }; + + std::variant result; +}; + +struct join_node_response_params { + // The topology coordinator accepts and wants to add the joining node + // to group 0 and to the cluster in general. + struct accepted {}; + + // The topology coordinator rejects the node, most likely due to some + // parameters being incorrect or incompatible with the cluster. + struct rejected { + sstring reason; + }; + + std::variant response; +}; + +struct join_node_response_result {}; + +}