/* * * Modified by ScyllaDB * Copyright (C) 2022-present ScyllaDB */ /* * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0) */ #pragma once #include #include #include #include #include #include #include #include #include "locator/types.hh" #include "inet_address_vectors.hh" using namespace seastar; namespace locator { class topology; } namespace std { std::ostream& operator<<(std::ostream& out, const locator::topology&); } namespace locator { class node; using node_holder = std::unique_ptr; using shard_id = seastar::shard_id; class node { public: using this_node = bool_class; using idx_type = int; enum class state { none = 0, bootstrapping, // (joining) replacing, // (joining) normal, being_decommissioned, // (leaving) being_removed, // (leaving) being_replaced, // (leaving) left // after decommissioned, removed, replaced }; private: const locator::topology* _topology; locator::host_id _host_id; inet_address _endpoint; endpoint_dc_rack _dc_rack; state _state; shard_id _shard_count = 0; // Is this node the `localhost` instance this_node _is_this_node; idx_type _idx = -1; public: node(const locator::topology* topology, locator::host_id id, inet_address endpoint, endpoint_dc_rack dc_rack, state state, shard_id shard_count = 0, this_node is_this_node = this_node::no, idx_type idx = -1); node(const node&) = delete; node(node&&) = delete; const locator::topology* topology() const noexcept { return _topology; } const locator::host_id& host_id() const noexcept { return _host_id; } const inet_address& endpoint() const noexcept { return _endpoint; } const endpoint_dc_rack& dc_rack() const noexcept { return _dc_rack; } // Is this "localhost"? this_node is_this_node() const noexcept { return _is_this_node; } // idx < 0 means "unassigned" idx_type idx() const noexcept { return _idx; } state get_state() const noexcept { return _state; } bool is_joining() const noexcept { switch (_state) { case state::bootstrapping: case state::replacing: return true; default: return false; } } bool is_normal() const noexcept { return _state == state::normal; } bool is_leaving() const noexcept { switch (_state) { case state::being_decommissioned: case state::being_removed: case state::being_replaced: return true; default: return false; } } bool left() const noexcept { return _state == state::left; } shard_id get_shard_count() const noexcept { return _shard_count; } static std::string to_string(state); private: static node_holder make(const locator::topology* topology, locator::host_id id, inet_address endpoint, endpoint_dc_rack dc_rack, state state, shard_id shard_count = 0, node::this_node is_this_node = this_node::no, idx_type idx = -1); node_holder clone() const; void set_topology(const locator::topology* topology) noexcept { _topology = topology; } void set_idx(idx_type idx) noexcept { _idx = idx; } void set_state(state state) noexcept { _state = state; } void set_shard_count(shard_id shard_count) noexcept { _shard_count = shard_count; } friend class topology; }; class topology { public: struct config { inet_address this_endpoint; host_id this_host_id; endpoint_dc_rack local_dc_rack; bool disable_proximity_sorting = false; bool operator==(const config&) const = default; }; topology(config cfg); topology(topology&&) noexcept; topology& operator=(topology&&) noexcept; future clone_gently() const; future<> clear_gently() noexcept; public: const config& get_config() const noexcept { return _cfg; } void set_host_id_cfg(host_id this_host_id) { _cfg.this_host_id = this_host_id; } const node* this_node() const noexcept { return _this_node; } // Adds a node with given host_id, endpoint, and DC/rack. const node* add_node(host_id id, const inet_address& ep, const endpoint_dc_rack& dr, node::state state, shard_id shard_count = 0); // Optionally updates node's current host_id, endpoint, or DC/rack. // Note: the host_id may be updated from null to non-null after a new node gets a new, random host_id, // or a peer node host_id may be updated when the node is replaced with another node using the same ip address. const node* update_node(node* node, std::optional opt_id, std::optional opt_ep, std::optional opt_dr, std::optional opt_st, std::optional opt_shard_count = std::nullopt); // Removes a node using its host_id // Returns true iff the node was found and removed. bool remove_node(host_id id); // Looks up a node by its host_id. // Returns a pointer to the node if found, or nullptr otherwise. const node* find_node(host_id id) const noexcept; const node& get_node(host_id id) const { auto n = find_node(id); if (!n) { throw_with_backtrace(format("Node {} not found in topology", id)); } return *n; }; // Looks up a node by its inet_address. // Returns a pointer to the node if found, or nullptr otherwise. const node* find_node(const inet_address& ep) const noexcept; // Finds a node by its index // Returns a pointer to the node if found, or nullptr otherwise. const node* find_node(node::idx_type idx) const noexcept; // Returns true if a node with given host_id is found bool has_node(host_id id) const noexcept; bool has_node(inet_address id) const noexcept; /** * Stores current DC/rack assignment for ep * * Adds or updates a node with given endpoint */ const node* add_or_update_endpoint(inet_address ep, std::optional opt_id, std::optional opt_dr, std::optional opt_st, std::optional shard_count = std::nullopt); // Legacy entry point from token_metadata::update_topology const node* add_or_update_endpoint(inet_address ep, endpoint_dc_rack dr, std::optional opt_st) { return add_or_update_endpoint(ep, std::nullopt, std::move(dr), std::move(opt_st), std::nullopt); } const node* add_or_update_endpoint(inet_address ep, host_id id) { return add_or_update_endpoint(ep, id, std::nullopt, std::nullopt, std::nullopt); } /** * Removes current DC/rack assignment for ep * Returns true if the node was found and removed. */ bool remove_endpoint(inet_address ep); /** * Returns true iff contains given endpoint. */ bool has_endpoint(inet_address) const; const std::unordered_map>& get_datacenter_endpoints() const { return _dc_endpoints; } const std::unordered_map>>& get_datacenter_racks() const { return _dc_racks; } const std::unordered_set& get_datacenters() const noexcept { return _datacenters; } // Get dc/rack location of this node const endpoint_dc_rack& get_location() const noexcept { return _this_node ? _this_node->dc_rack() : _cfg.local_dc_rack; } // Get dc/rack location of a node identified by host_id // The specified node must exist. const endpoint_dc_rack& get_location(host_id id) const { return find_node(id)->dc_rack(); } // Get dc/rack location of a node identified by endpoint // The specified node must exist. const endpoint_dc_rack& get_location(const inet_address& ep) const; // Get datacenter of this node const sstring& get_datacenter() const noexcept { return get_location().dc; } // Get datacenter of a node identified by host_id // The specified node must exist. const sstring& get_datacenter(host_id id) const { return get_location(id).dc; } // Get datacenter of a node identified by endpoint // The specified node must exist. const sstring& get_datacenter(inet_address ep) const { return get_location(ep).dc; } // Get rack of this node const sstring& get_rack() const noexcept { return get_location().rack; } // Get rack of a node identified by host_id // The specified node must exist. const sstring& get_rack(host_id id) const { return get_location(id).rack; } // Get rack of a node identified by endpoint // The specified node must exist. const sstring& get_rack(inet_address ep) const { return get_location(ep).rack; } auto get_local_dc_filter() const noexcept { return [ this, local_dc = get_datacenter() ] (inet_address ep) { return get_datacenter(ep) == local_dc; }; }; template inline size_t count_local_endpoints(const Range& endpoints) const { return std::count_if(endpoints.begin(), endpoints.end(), get_local_dc_filter()); } /** * This method will sort the List by proximity to the given * address. */ void sort_by_proximity(inet_address address, inet_address_vector_replica_set& addresses) const; void for_each_node(std::function func) const; private: bool is_configured_this_node(const node&) const; const node* add_node(node_holder node); void remove_node(const node* node); static std::string debug_format(const node*); void index_node(const node* node); void unindex_node(const node* node); node_holder pop_node(const node* node); static node* make_mutable(const node* nptr) { return const_cast(nptr); } /** * compares two endpoints in relation to the target endpoint, returning as * Comparator.compare would * * The closest nodes to a given node are: * 1. The node itself * 2. Nodes in the same RACK as the reference node * 3. Nodes in the same DC as the reference node */ std::weak_ordering compare_endpoints(const inet_address& address, const inet_address& a1, const inet_address& a2) const; unsigned _shard; config _cfg; const node* _this_node = nullptr; std::vector _nodes; std::unordered_map _nodes_by_host_id; std::unordered_map _nodes_by_endpoint; std::unordered_map> _dc_nodes; std::unordered_map>> _dc_rack_nodes; /** multi-map: DC -> endpoints in that DC */ std::unordered_map> _dc_endpoints; /** map: DC -> (multi-map: rack -> endpoints in that rack) */ std::unordered_map>> _dc_racks; bool _sort_by_proximity = true; // pre-calculated std::unordered_set _datacenters; void calculate_datacenters(); const std::unordered_map& get_nodes_by_endpoint() const noexcept { return _nodes_by_endpoint; }; friend class token_metadata_impl; public: void test_compare_endpoints(const inet_address& address, const inet_address& a1, const inet_address& a2) const; friend std::ostream& std::operator<<(std::ostream& out, const topology&); }; } // namespace locator namespace std { std::ostream& operator<<(std::ostream& out, const locator::node& node); std::ostream& operator<<(std::ostream& out, const locator::node::state& state); } // namespace std template <> struct fmt::formatter : fmt::formatter { template auto format(const locator::node& node, FormatContext& ctx) const { return fmt::format_to(ctx.out(), "{}/{}", node.host_id(), node.endpoint()); } }; template <> struct fmt::formatter : fmt::formatter { template auto format(const locator::node::state& state, FormatContext& ctx) const { return fmt::format_to(ctx.out(), "{}", locator::node::to_string(state)); } };