From 5d1f60439a845c1ea91fce17de35b879c8d642e0 Mon Sep 17 00:00:00 2001 From: "Raphael S. Carvalho" Date: Mon, 14 Aug 2023 17:11:52 -0300 Subject: [PATCH] token_metadata: Add this_host_id to topology config The motivation is that token_metadata::get_my_id() is not available early in the bootstrap process, as raft topology is pulled later than new tables are registered and created, and this node is added to topology even later. To allow creation of compaction groups to retrieve "my id" from token metadata early, initialization will now feed local id into topology config which is immutable for each node anyway. Signed-off-by: Raphael S. Carvalho --- locator/token_metadata.cc | 2 +- locator/topology.hh | 5 +++++ main.cc | 14 +++++++++----- test/boost/tablets_test.cc | 2 +- test/lib/cql_test_env.cc | 4 ++++ 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/locator/token_metadata.cc b/locator/token_metadata.cc index fcdb9cc193..921cca71bd 100644 --- a/locator/token_metadata.cc +++ b/locator/token_metadata.cc @@ -334,7 +334,7 @@ token_metadata::tokens_iterator& token_metadata::tokens_iterator::operator++() { } host_id token_metadata::get_my_id() const { - return get_host_id(utils::fb_utilities::get_broadcast_address()); + return get_topology().get_config().this_host_id; } inline diff --git a/locator/topology.hh b/locator/topology.hh index 3c2ef3945b..1432f76def 100644 --- a/locator/topology.hh +++ b/locator/topology.hh @@ -161,6 +161,7 @@ class topology { public: struct config { inet_address this_endpoint; + host_id this_host_id; endpoint_dc_rack local_dc_rack; bool disable_proximity_sorting = false; @@ -177,6 +178,10 @@ public: public: const config& get_config() const noexcept { return _cfg; } + void set_host_id_cfg(host_id this_host_id) { + _cfg.this_host_id = this_host_id; + } + const node* this_node() const noexcept { return _this_node; } diff --git a/main.cc b/main.cc index a281236ed7..4ffea3e95d 100644 --- a/main.cc +++ b/main.cc @@ -1103,6 +1103,15 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl replica::distributed_loader::init_system_keyspace(sys_ks, erm_factory, db, *cfg, system_table_load_phase::phase1).get(); cfg->host_id = sys_ks.local().load_local_host_id().get0(); + shared_token_metadata::mutate_on_all_shards(token_metadata, [hostid = cfg->host_id, endpoint = utils::fb_utilities::get_broadcast_address()] (locator::token_metadata& tm) { + // Makes local host id available in topology cfg as soon as possible. + // Raft topology discard the endpoint-to-id map, so the local id can + // still be found in the config. + tm.get_topology().set_host_id_cfg(hostid); + tm.get_topology().add_or_update_endpoint(endpoint, hostid); + return make_ready_future<>(); + }).get(); + netw::messaging_service::config mscfg; mscfg.id = cfg->host_id; @@ -1291,11 +1300,6 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl // engine().at_exit([&qp] { return qp.stop(); }); sstables::init_metrics().get(); - shared_token_metadata::mutate_on_all_shards(token_metadata, [hostid = cfg->host_id, endpoint = utils::fb_utilities::get_broadcast_address()] (locator::token_metadata& tm) { - tm.get_topology().add_or_update_endpoint(endpoint, hostid); - return make_ready_future<>(); - }).get(); - supervisor::notify("initializing batchlog manager"); db::batchlog_manager_config bm_cfg; bm_cfg.write_request_timeout = cfg->write_request_timeout_in_ms() * 1ms; diff --git a/test/boost/tablets_test.cc b/test/boost/tablets_test.cc index b66f24c351..f3ef0ab73d 100644 --- a/test/boost/tablets_test.cc +++ b/test/boost/tablets_test.cc @@ -433,7 +433,7 @@ SEASTAR_TEST_CASE(test_sharder) { auto table1 = table_id(utils::UUID_gen::get_time_UUID()); - token_metadata tokm(token_metadata::config{}); + token_metadata tokm(token_metadata::config{ .topo_cfg{ .this_host_id = h1 } }); tokm.get_topology().add_or_update_endpoint(utils::fb_utilities::get_broadcast_address(), h1); std::vector tablet_ids; diff --git a/test/lib/cql_test_env.cc b/test/lib/cql_test_env.cc index ef0836642e..6d4577c75d 100644 --- a/test/lib/cql_test_env.cc +++ b/test/lib/cql_test_env.cc @@ -669,6 +669,10 @@ private: if (!cfg->host_id) { cfg->host_id = _sys_ks.local().load_local_host_id().get0(); } + locator::shared_token_metadata::mutate_on_all_shards(_token_metadata, [hostid = cfg->host_id] (locator::token_metadata& tm) { + tm.get_topology().set_host_id_cfg(hostid); + return make_ready_future<>(); + }).get(); // don't start listening so tests can be run in parallel _ms.start(cfg->host_id, listen, std::move(7000)).get();