token_metadata: Add this_host_id to topology config

The motivation is that token_metadata::get_my_id() is not available
early in the bootstrap process, as raft topology is pulled later
than new tables are registered and created, and this node is added
to topology even later.

To allow creation of compaction groups to retrieve "my id" from
token metadata early, initialization will now feed local id
into topology config which is immutable for each node anyway.

Signed-off-by: Raphael S. Carvalho <raphaelsc@scylladb.com>

locator/token_metadata.cc

@@ -334,7 +334,7 @@ token_metadata::tokens_iterator& token_metadata::tokens_iterator::operator++() {
 }
 
 host_id token_metadata::get_my_id() const {
-    return get_host_id(utils::fb_utilities::get_broadcast_address());
+    return get_topology().get_config().this_host_id;
 }
 
 inline

locator/topology.hh

@@ -161,6 +161,7 @@ class topology {
 public:
     struct config {
         inet_address this_endpoint;
+        host_id this_host_id;
         endpoint_dc_rack local_dc_rack;
         bool disable_proximity_sorting = false;
 
@@ -177,6 +178,10 @@ public:
 public:
     const config& get_config() const noexcept { return _cfg; }
 
+    void set_host_id_cfg(host_id this_host_id) {
+        _cfg.this_host_id = this_host_id;
+    }
+
     const node* this_node() const noexcept {
         return _this_node;
     }

main.cc

@@ -1103,6 +1103,15 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl
             replica::distributed_loader::init_system_keyspace(sys_ks, erm_factory, db, *cfg, system_table_load_phase::phase1).get();
             cfg->host_id = sys_ks.local().load_local_host_id().get0();
 
+          shared_token_metadata::mutate_on_all_shards(token_metadata, [hostid = cfg->host_id, endpoint = utils::fb_utilities::get_broadcast_address()] (locator::token_metadata& tm) {
+              // Makes local host id available in topology cfg as soon as possible.
+              // Raft topology discard the endpoint-to-id map, so the local id can
+              // still be found in the config.
+              tm.get_topology().set_host_id_cfg(hostid);
+              tm.get_topology().add_or_update_endpoint(endpoint, hostid);
+              return make_ready_future<>();
+          }).get();
+
             netw::messaging_service::config mscfg;
 
             mscfg.id = cfg->host_id;
@@ -1291,11 +1300,6 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl
             // engine().at_exit([&qp] { return qp.stop(); });
             sstables::init_metrics().get();
 
-            shared_token_metadata::mutate_on_all_shards(token_metadata, [hostid = cfg->host_id, endpoint = utils::fb_utilities::get_broadcast_address()] (locator::token_metadata& tm) {
-                tm.get_topology().add_or_update_endpoint(endpoint, hostid);
-                return make_ready_future<>();
-            }).get();
-
             supervisor::notify("initializing batchlog manager");
             db::batchlog_manager_config bm_cfg;
             bm_cfg.write_request_timeout = cfg->write_request_timeout_in_ms() * 1ms;

test/boost/tablets_test.cc

@@ -433,7 +433,7 @@ SEASTAR_TEST_CASE(test_sharder) {
 
         auto table1 = table_id(utils::UUID_gen::get_time_UUID());
 
-        token_metadata tokm(token_metadata::config{});
+        token_metadata tokm(token_metadata::config{ .topo_cfg{ .this_host_id = h1 } });
         tokm.get_topology().add_or_update_endpoint(utils::fb_utilities::get_broadcast_address(), h1);
 
         std::vector<tablet_id> tablet_ids;

test/lib/cql_test_env.cc

@@ -669,6 +669,10 @@ private:
             if (!cfg->host_id) {
                 cfg->host_id = _sys_ks.local().load_local_host_id().get0();
             }
+            locator::shared_token_metadata::mutate_on_all_shards(_token_metadata, [hostid = cfg->host_id] (locator::token_metadata& tm) {
+                tm.get_topology().set_host_id_cfg(hostid);
+                return make_ready_future<>();
+            }).get();
 
             // don't start listening so tests can be run in parallel
             _ms.start(cfg->host_id, listen, std::move(7000)).get();