From 7ddf8963f598edc8b3dc13ee5cadd696eb021910 Mon Sep 17 00:00:00 2001 From: Asias He Date: Thu, 19 Nov 2015 16:25:29 +0800 Subject: [PATCH] config: Enable broadcast_rpc_address option With this patch, start two nodes node 1: scylla --rpc-address 127.0.0.1 --broadcast-rpc-address 127.0.0.11 node 2: scylla --rpc-address 127.0.0.2 --broadcast-rpc-address 127.0.0.12 On node 1: cqlsh> SELECT rpc_address from system.peers; rpc_address ------------- 127.0.0.12 which means client should use this address to connect node 2 for cql and thrift protocol. --- db/config.hh | 2 +- locator/ec2_multi_region_snitch.cc | 3 +-- main.cc | 11 +++++++++++ service/storage_service.cc | 3 +-- tests/cql_test_env.cc | 1 + tests/ec2_snitch_test.cc | 1 + tests/gossip.cc | 1 + tests/gossiping_property_file_snitch_test.cc | 1 + tests/network_topology_strategy_test.cc | 2 ++ tests/snitch_reset_test.cc | 1 + 10 files changed, 21 insertions(+), 5 deletions(-) diff --git a/db/config.hh b/db/config.hh index e104a9d41c..c0af6160fc 100644 --- a/db/config.hh +++ b/db/config.hh @@ -560,7 +560,7 @@ public: ) \ /* RPC (remote procedure call) settings */ \ /* Settings for configuring and tuning client connections. */ \ - val(broadcast_rpc_address, sstring, /* unset */, Unused, \ + val(broadcast_rpc_address, sstring, /* unset */, Used, \ "RPC address to broadcast to drivers and other Cassandra nodes. This cannot be set to 0.0.0.0. If blank, it is set to the value of the rpc_address or rpc_interface. If rpc_address or rpc_interfaceis set to 0.0.0.0, this property must be set.\n" \ ) \ val(rpc_port, uint16_t, 9160, Used, \ diff --git a/locator/ec2_multi_region_snitch.cc b/locator/ec2_multi_region_snitch.cc index 53db54db78..3b0277ad8a 100644 --- a/locator/ec2_multi_region_snitch.cc +++ b/locator/ec2_multi_region_snitch.cc @@ -61,8 +61,7 @@ future<> ec2_multi_region_snitch::start() { // value to a public address in cassandra.yaml. // utils::fb_utilities::set_broadcast_address(local_public_address); - //DatabaseDescriptor.setBroadcastRpcAddress(local_public_address); - // + utils::fb_utilities::set_broadcast_rpc_address(local_public_address); return aws_api_call(AWS_QUERY_SERVER_ADDR, PRIVATE_IP_QUERY_REQ).then( [this] (sstring priv_addr) { diff --git a/main.cc b/main.cc index 763bcffd2c..acbe1cb1cc 100644 --- a/main.cc +++ b/main.cc @@ -206,6 +206,7 @@ int main(int ac, char** av) { sstring api_address = cfg->api_address() != "" ? cfg->api_address() : rpc_address; auto seed_provider= cfg->seed_provider(); sstring broadcast_address = cfg->broadcast_address(); + sstring broadcast_rpc_address = cfg->broadcast_rpc_address(); if (!broadcast_address.empty()) { utils::fb_utilities::set_broadcast_address(broadcast_address); @@ -216,6 +217,16 @@ int main(int ac, char** av) { throw bad_configuration_error(); } + if (!broadcast_rpc_address.empty()) { + utils::fb_utilities::set_broadcast_rpc_address(broadcast_rpc_address); + } else { + if (rpc_address == "0.0.0.0") { + startlog.error("If rpc_address is set to a wildcard address {}, then you must set broadcast_rpc_address to a value other than {}", rpc_address, rpc_address); + throw bad_configuration_error(); + } + utils::fb_utilities::set_broadcast_rpc_address(rpc_address); + } + using namespace locator; return i_endpoint_snitch::create_snitch(cfg->endpoint_snitch()).then([] { // #293 - do not stop anything diff --git a/service/storage_service.cc b/service/storage_service.cc index 0311100a9f..9ee2365e59 100644 --- a/service/storage_service.cc +++ b/service/storage_service.cc @@ -202,8 +202,7 @@ future<> storage_service::prepare_to_join() { return db::system_keyspace::get_local_host_id(); }).then([this, app_states] (auto local_host_id) mutable { _token_metadata.update_host_id(local_host_id, this->get_broadcast_address()); - // FIXME: DatabaseDescriptor.getBroadcastRpcAddress() - auto broadcast_rpc_address = this->get_broadcast_address(); + auto broadcast_rpc_address = utils::fb_utilities::get_broadcast_rpc_address(); app_states->emplace(gms::application_state::NET_VERSION, value_factory.network_version()); app_states->emplace(gms::application_state::HOST_ID, value_factory.host_id(local_host_id)); app_states->emplace(gms::application_state::RPC_ADDRESS, value_factory.rpcaddress(broadcast_rpc_address)); diff --git a/tests/cql_test_env.cc b/tests/cql_test_env.cc index 4a198b472c..a019eadb27 100644 --- a/tests/cql_test_env.cc +++ b/tests/cql_test_env.cc @@ -283,6 +283,7 @@ public: } return seastar::async([this] { utils::fb_utilities::set_broadcast_address(gms::inet_address("localhost")); + utils::fb_utilities::set_broadcast_rpc_address(gms::inet_address("localhost")); locator::i_endpoint_snitch::create_snitch("SimpleSnitch").get(); auto db = ::make_shared>(); init_once(db).get(); diff --git a/tests/ec2_snitch_test.cc b/tests/ec2_snitch_test.cc index 913bd9dc26..97f432e603 100644 --- a/tests/ec2_snitch_test.cc +++ b/tests/ec2_snitch_test.cc @@ -43,6 +43,7 @@ future<> one_test(const std::string& property_fname, bool exp_result) { fname /= path(property_fname); utils::fb_utilities::set_broadcast_address(gms::inet_address("localhost")); + utils::fb_utilities::set_broadcast_rpc_address(gms::inet_address("localhost")); return i_endpoint_snitch::create_snitch( "EC2Snitch", diff --git a/tests/gossip.cc b/tests/gossip.cc index d04bc8a68f..b55f41f495 100644 --- a/tests/gossip.cc +++ b/tests/gossip.cc @@ -65,6 +65,7 @@ int main(int ac, char ** av) { logging::logger_registry().set_logger_level("gossip", logging::log_level::trace); const gms::inet_address listen = gms::inet_address(config["listen-address"].as()); utils::fb_utilities::set_broadcast_address(listen); + utils::fb_utilities::set_broadcast_rpc_address(listen); auto vv = std::make_shared(); locator::i_endpoint_snitch::create_snitch("SimpleSnitch").then([&db] { return service::init_storage_service(db); diff --git a/tests/gossiping_property_file_snitch_test.cc b/tests/gossiping_property_file_snitch_test.cc index 04ddd7749c..ea5413d3c6 100644 --- a/tests/gossiping_property_file_snitch_test.cc +++ b/tests/gossiping_property_file_snitch_test.cc @@ -43,6 +43,7 @@ future<> one_test(const std::string& property_fname, bool exp_result) { fname /= path(property_fname); utils::fb_utilities::set_broadcast_address(gms::inet_address("localhost")); + utils::fb_utilities::set_broadcast_rpc_address(gms::inet_address("localhost")); return i_endpoint_snitch::create_snitch( "org.apache.cassandra.locator.GossipingPropertyFileSnitch", diff --git a/tests/network_topology_strategy_test.cc b/tests/network_topology_strategy_test.cc index cfbfa67b70..e6c4d85edc 100644 --- a/tests/network_topology_strategy_test.cc +++ b/tests/network_topology_strategy_test.cc @@ -157,6 +157,7 @@ void full_ring_check(const std::vector& ring_points, future<> simple_test() { utils::fb_utilities::set_broadcast_address(gms::inet_address("localhost")); + utils::fb_utilities::set_broadcast_rpc_address(gms::inet_address("localhost")); // Create the RackInferringSnitch return i_endpoint_snitch::create_snitch("RackInferringSnitch").then( @@ -230,6 +231,7 @@ future<> simple_test() { future<> heavy_origin_test() { utils::fb_utilities::set_broadcast_address(gms::inet_address("localhost")); + utils::fb_utilities::set_broadcast_rpc_address(gms::inet_address("localhost")); // Create the RackInferringSnitch return i_endpoint_snitch::create_snitch("RackInferringSnitch").then( diff --git a/tests/snitch_reset_test.cc b/tests/snitch_reset_test.cc index ec29dc1f63..0404cb01ca 100644 --- a/tests/snitch_reset_test.cc +++ b/tests/snitch_reset_test.cc @@ -38,6 +38,7 @@ future<> one_test(const std::string& property_fname1, using namespace boost::filesystem; utils::fb_utilities::set_broadcast_address(gms::inet_address("localhost")); + utils::fb_utilities::set_broadcast_rpc_address(gms::inet_address("localhost")); printf("Testing %s and %s property files. Expected result is %s\n", property_fname1.c_str(), property_fname2.c_str(),