From 6b246dc119099629f13f1b02479eb752507ae283 Mon Sep 17 00:00:00 2001
From: Takuya ASADA <syuu@scylladb.com>
Date: Mon, 3 Oct 2022 14:31:20 +0900
Subject: [PATCH] locator::ec2_snitch: Retry HTTP request to EC2 instance
 metadata service

EC2 instance metadata service can be busy, ret's retry to connect with
interval, just like we do in scylla-machine-image.

Fixes #10250

Signed-off-by: Takuya ASADA <syuu@scylladb.com>

Closes #11688
---
 locator/ec2_snitch.cc | 26 ++++++++++++++++++++++++++
 locator/ec2_snitch.hh |  3 +++
 2 files changed, 29 insertions(+)

diff --git a/locator/ec2_snitch.cc b/locator/ec2_snitch.cc
index 64fb3619f3..f6234528be 100644
--- a/locator/ec2_snitch.cc
+++ b/locator/ec2_snitch.cc
@@ -1,5 +1,7 @@
 #include "locator/ec2_snitch.hh"
 #include <seastar/core/seastar.hh>
+#include <seastar/core/sleep.hh>
+#include <seastar/core/do_with.hh>
 
 #include <boost/algorithm/string/classification.hpp>
 #include <boost/algorithm/string/split.hpp>
@@ -65,6 +67,30 @@ future<> ec2_snitch::start() {
 }
 
 future<sstring> ec2_snitch::aws_api_call(sstring addr, uint16_t port, sstring cmd, std::optional<sstring> token) {
+    return do_with(int(0), [this, addr, port, cmd, token] (int& i) {
+        return repeat_until_value([this, addr, port, cmd, token, &i]() -> future<std::optional<sstring>> {
+            ++i;
+            return aws_api_call_once(addr, port, cmd, token).then([] (auto res) {
+                return make_ready_future<std::optional<sstring>>(std::move(res));
+            }).handle_exception([&i] (auto ep) {
+                try {
+                    std::rethrow_exception(ep);
+                } catch (const std::system_error &e) {
+                    logger().error(e.what());
+                    if (i >= AWS_API_CALL_RETRIES - 1) {
+                        logger().error("Maximum number of retries exceeded");
+                        throw e;
+                    }
+                }
+                return sleep(AWS_API_CALL_RETRY_INTERVAL).then([] {
+                    return make_ready_future<std::optional<sstring>>(std::nullopt);
+                });
+            });
+        });
+    });
+}
+
+future<sstring> ec2_snitch::aws_api_call_once(sstring addr, uint16_t port, sstring cmd, std::optional<sstring> token) {
     return connect(socket_address(inet_address{addr}, port))
     .then([this, addr, cmd, token] (connected_socket fd) {
         _sd = std::move(fd);
diff --git a/locator/ec2_snitch.hh b/locator/ec2_snitch.hh
index 88ee1fc769..41a980ac82 100644
--- a/locator/ec2_snitch.hh
+++ b/locator/ec2_snitch.hh
@@ -17,6 +17,8 @@ public:
     static constexpr const char* ZONE_NAME_QUERY_REQ = "/latest/meta-data/placement/availability-zone";
     static constexpr const char* AWS_QUERY_SERVER_ADDR = "169.254.169.254";
     static constexpr uint16_t AWS_QUERY_SERVER_PORT = 80;
+    static constexpr int AWS_API_CALL_RETRIES = 5;
+    static constexpr auto AWS_API_CALL_RETRY_INTERVAL = std::chrono::seconds{5};
 
     ec2_snitch(const snitch_config&);
     virtual future<> start() override;
@@ -33,5 +35,6 @@ private:
     output_stream<char> _out;
     http_response_parser _parser;
     sstring _req;
+    future<sstring> aws_api_call_once(sstring addr, uint16_t port, const sstring cmd, std::optional<sstring> token);
 };
 } // namespace locator