abstract_replication_strategy: Add allow_remove_node_being_replaced_from_natural_endpoints

Decide if the replication strategy allow removing the node being replaced from
the natural endpoints when a node is being replaced in the cluster.
LocalStrategy is the not allowed to do so because it always returns the node
itself as the natural_endpoints and the node will not appear in the
pending_endpoints.

It is needed by the "Make replacing node take writes" work.

Refs: #5482

locator/abstract_replication_strategy.hh

@@ -98,6 +98,12 @@ public:
     virtual void validate_options() const = 0;
     virtual std::optional<std::set<sstring>> recognized_options() const = 0;
     virtual size_t get_replication_factor() const = 0;
+    // Decide if the replication strategy allow removing the node being
+    // replaced from the natural endpoints when a node is being replaced in the
+    // cluster. LocalStrategy is the not allowed to do so because it always
+    // returns the node itself as the natural_endpoints and the node will not
+    // appear in the pending_endpoints.
+    virtual bool allow_remove_node_being_replaced_from_natural_endpoints() const = 0;
     uint64_t get_cache_hits_count() const { return _cache_hits_count; }
     replication_strategy_type get_type() const { return _my_type; }
 

locator/everywhere_replication_strategy.hh

@@ -61,5 +61,9 @@ public:
     virtual size_t get_replication_factor() const override {
         return _token_metadata.get_all_endpoints_count();
     }
+
+    virtual bool allow_remove_node_being_replaced_from_natural_endpoints() const override {
+        return true;
+    }
 };
 }

locator/local_strategy.hh

@@ -51,6 +51,11 @@ public:
     virtual void validate_options() const override;
 
     virtual std::optional<std::set<sstring>> recognized_options() const override;
+
+    virtual bool allow_remove_node_being_replaced_from_natural_endpoints() const override {
+        return false;
+    }
+
 };
 
 }

locator/network_topology_strategy.hh

@@ -66,6 +66,10 @@ public:
         return _datacenteres;
     }
 
+    virtual bool allow_remove_node_being_replaced_from_natural_endpoints() const override {
+        return true;
+    }
+
 protected:
     /**
      * calculate endpoints in one pass through the tokens by tracking our

locator/simple_strategy.hh

@@ -37,6 +37,9 @@ public:
     virtual size_t get_replication_factor() const override;
     virtual void validate_options() const override;
     virtual std::optional<std::set<sstring>> recognized_options() const override;
+    virtual bool allow_remove_node_being_replaced_from_natural_endpoints() const override {
+        return true;
+    }
 private:
     size_t _replication_factor = 1;
 };