topology: Put local dc/rack on topology early

Startup code needs to know the dc/rack of the local node early, way
before nodes starts any communication with the ring. This information is
available when snitch activates, but it starts _after_ token-metadata,
so the only way to put local dc/rack in topology is via a startup-time
special API call. This new init_local_endpoint() is temporary and will
be removed later in this set

Signed-off-by: Pavel Emelyanov <xemul@scylladb.com>

locator/token_metadata.cc

@@ -282,6 +282,7 @@ public:
     size_t count_normal_token_owners() const;
 private:
     future<> update_normal_token_owners();
+    void init_local_endpoint(endpoint_dc_rack) noexcept;
 
 public:
     // returns empty vector if keyspace_name not found.
@@ -1423,4 +1424,20 @@ future<> shared_token_metadata::mutate_token_metadata(seastar::noncopyable_funct
     set(make_token_metadata_ptr(std::move(tm)));
 }
 
+void shared_token_metadata::init_local_endpoint(endpoint_dc_rack dr) noexcept {
+    _shared->init_local_endpoint(std::move(dr));
+}
+
+void token_metadata::init_local_endpoint(endpoint_dc_rack dr) noexcept {
+    _impl->init_local_endpoint(std::move(dr));
+}
+
+void token_metadata_impl::init_local_endpoint(endpoint_dc_rack dr) noexcept {
+    _topology.init_local_endpoint(std::move(dr));
+}
+
+void topology::init_local_endpoint(endpoint_dc_rack dr) noexcept {
+    _pending_locations[utils::fb_utilities::get_broadcast_address()] = std::move(dr);
+}
+
 } // namespace locator

locator/token_metadata.hh

@@ -109,6 +109,8 @@ public:
         _sort_by_proximity = false;
     }
 
+    void init_local_endpoint(endpoint_dc_rack) noexcept;
+
 private:
     /**
      * compares two endpoints in relation to the target endpoint, returning as
@@ -166,8 +168,10 @@ private:
 
         friend class token_metadata_impl;
     };
+
 public:
     token_metadata(config cfg);
+    void init_local_endpoint(endpoint_dc_rack) noexcept;
     explicit token_metadata(std::unique_ptr<token_metadata_impl> impl);
     token_metadata(token_metadata&&) noexcept; // Can't use "= default;" - hits some static_assert in unique_ptr
     token_metadata& operator=(token_metadata&&) noexcept;
@@ -381,6 +385,9 @@ public:
         return _lock_func();
     }
 
+    // FIXME -- snitch should start early and provide this info via constructor
+    void init_local_endpoint(endpoint_dc_rack) noexcept;
+
     // mutate_token_metadata acquires the shared_token_metadata lock,
     // clones the token_metadata (using clone_async)
     // and calls an asynchronous functor on

main.cc

@@ -924,6 +924,12 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl
                 }).get();
             }
 
+            // FIXME -- token metadata should get local DC/RACK via constructor,
+            // but snitch cannot (yet) start early enough to provide it
+            token_metadata.invoke_on_all([&snitch, &broadcast_addr] (auto& tm) {
+                tm.init_local_endpoint({ snitch.local()->get_datacenter(broadcast_addr), snitch.local()->get_rack(broadcast_addr) });
+            }).get();
+
             static direct_fd_clock fd_clock;
             static sharded<direct_failure_detector::failure_detector> fd;
             supervisor::notify("starting direct failure detector service");

test/lib/cql_test_env.cc

@@ -609,6 +609,11 @@ public:
             auto stop_snitch = defer([&snitch] { snitch.stop().get(); });
             snitch.invoke_on_all(&locator::snitch_ptr::start).get();
 
+            token_metadata.invoke_on_all([&snitch] (auto& tm) {
+                auto ep = utils::fb_utilities::get_broadcast_address();
+                tm.init_local_endpoint({ snitch.local()->get_datacenter(ep), snitch.local()->get_rack(ep) });
+            }).get();
+
             distributed<service::storage_proxy>& proxy = service::get_storage_proxy();
             distributed<service::migration_manager> mm;
             sharded<cql3::cql_config> cql_config;