From 77b1db475ce63a6f4c77183299b7f1cbea16a49e Mon Sep 17 00:00:00 2001 From: Asias He Date: Mon, 9 May 2022 10:30:44 +0800 Subject: [PATCH] locator: Do not enforce public ip address for broadcast_rpc_address Reported by Felipe Cardeneti: - Create a 2-node Scylla cluster w/ Ec2MultiRegionSnitch - Check system.peers table Scylla (uses public address) ``` cqlsh> select peer,data_center,host_id,preferred_ip,rack,rpc_address,schema_version from system.peers; peer | data_center | host_id | preferred_ip | rack | rpc_address | schema_version ---------------+-------------+--------------------------------------+---------------+------+---------------+-------------------------------------- 18.216.98.219 | us-east-2 | d9443741-a12e-4bbb-91ce-9931cece589c | 172.31.43.122 | 2c | 18.216.98.219 | 95c3fca5-c463-3aba-98c6-1c0b3fac5b58 (1 rows) ``` Cassandra (uses local address): ``` cqlsh> SELECT peer,data_center,host_id,preferred_ip,rack,rpc_address,schema_version from system.peers; peer | data_center | host_id | preferred_ip | rack | rpc_address | schema_version ---------------+-------------+--------------------------------------+---------------+------------+---------------+-------------------------------------- 52.15.104.255 | us-east-2 | 42c0b717-775f-4998-a420-0388fe8b4e70 | 172.31.42.126 | us-east-2c | 172.31.42.126 | 2207c2a9-f598-3971-986b-2926e09e239d (1 rows) ``` Config diff: ``` cassandra.yaml:rpc_address: 0.0.0.0 cassandra.yaml:broadcast_rpc_address: 172.31.42.126 /etc/scylla/scylla.yaml:broadcast_rpc_address: 172.31.42.126 /etc/scylla/scylla.yaml:rpc_address: 0.0.0.0 ``` After this patch, if broadcast_rpc_address is unset, Ec2MultiRegionSnitch will use the public ip address to set broadcast_rpc_address. If broadcast_rpc_address is set, Ec2MultiRegionSnitch will not modify it. Fixes #10236 Closes #10519 --- locator/ec2_multi_region_snitch.cc | 7 +++++-- locator/ec2_multi_region_snitch.hh | 1 + locator/snitch_base.hh | 1 + main.cc | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/locator/ec2_multi_region_snitch.cc b/locator/ec2_multi_region_snitch.cc index f0b399cc47..b01bc2373f 100644 --- a/locator/ec2_multi_region_snitch.cc +++ b/locator/ec2_multi_region_snitch.cc @@ -20,7 +20,8 @@ static constexpr const char* PRIVATE_MAC_QUERY = "/latest/meta-data/network/inte namespace locator { ec2_multi_region_snitch::ec2_multi_region_snitch(const snitch_config& cfg) - : ec2_snitch(cfg) {} + : ec2_snitch(cfg) + , _broadcast_rpc_address_specified_by_user(cfg.broadcast_rpc_address_specified_by_user) {} future<> ec2_multi_region_snitch::start() { _state = snitch_state::initializing; @@ -61,7 +62,9 @@ future<> ec2_multi_region_snitch::start() { // value to a public address in cassandra.yaml. // utils::fb_utilities::set_broadcast_address(local_public_address); - utils::fb_utilities::set_broadcast_rpc_address(local_public_address); + if (!_broadcast_rpc_address_specified_by_user) { + utils::fb_utilities::set_broadcast_rpc_address(local_public_address); + } if (!local_public_address.addr().is_ipv6()) { sstring priv_addr = aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, PRIVATE_IP_QUERY_REQ).get0(); diff --git a/locator/ec2_multi_region_snitch.hh b/locator/ec2_multi_region_snitch.hh index 7e8db3b640..2fca3914e3 100644 --- a/locator/ec2_multi_region_snitch.hh +++ b/locator/ec2_multi_region_snitch.hh @@ -24,5 +24,6 @@ public: } private: sstring _local_private_address; + bool _broadcast_rpc_address_specified_by_user; }; } // namespace locator diff --git a/locator/snitch_base.hh b/locator/snitch_base.hh index e3bc9f5888..e37392c40a 100644 --- a/locator/snitch_base.hh +++ b/locator/snitch_base.hh @@ -45,6 +45,7 @@ struct snitch_config { sstring name = "SimpleSnitch"; sstring properties_file_name = ""; unsigned io_cpu_id = 0; + bool broadcast_rpc_address_specified_by_user = false; // GCE-specific sstring gce_meta_server_url = ""; diff --git a/main.cc b/main.cc index 9c49d61a5a..ac6d6e8347 100644 --- a/main.cc +++ b/main.cc @@ -870,6 +870,7 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl supervisor::notify("creating snitch"); snitch_config snitch_cfg; snitch_cfg.name = cfg->endpoint_snitch(); + snitch_cfg.broadcast_rpc_address_specified_by_user = !cfg->broadcast_rpc_address().empty(); sharded& snitch = i_endpoint_snitch::snitch_instance(); snitch.start(snitch_cfg, std::ref(gossiper)).get(); auto stop_snitch = defer_verbose_shutdown("snitch", [&snitch] {