test/nodetool: run nodetool tests using "unshare"

before this change, we use a random address when launching
rest_api_mock server, but there are chances that the randomly
picked address conflicts with an already-used address on the
host. and the subprocess fails right away with the returncode of
1 upon this failure, but we just continue on and check the readiness
of the already-dead server. actually, we've seen test failures
caused by the EADDRINUSE failure, and when we checked the readiness
of the rest_api_mock by sending HTTP request and reading the
response, what we had is not a JSON encoded response but a webpage,
which was likely the one returned by a minio server.

in this change, we

* specify the "launcher" option of nodetool
  test suite to "unshare", so that all its tests are launched
  in separated namespaces.
* use a random fixed address for the mock server, as the network
  namespaces are not shared anymore
* add an option in `nodetool/conftest.py`, so that it can optionally
  setup the lo network interface when it is launched in a separated
  new network namespace.

Fixes #16542
Signed-off-by: Kefu Chai <kefu.chai@scylladb.com>

test/nodetool/conftest.py

@@ -27,31 +27,51 @@ def pytest_addoption(parser):
                      " with --nodetool=cassandra, this should be the nodetool binary")
     parser.addoption('--jmx-path', action='store', default=None,
                      help="Path to the jmx binary, only used with --nodetool=cassandra")
+    parser.addoption('--run-within-unshare', action='store_true',
+                     help="Setup the 'lo' network if launched with unshare(1)")
 
 
 @pytest.fixture(scope="session")
-def rest_api_mock_server():
-    ip = f"127.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}"
-    port = random.randint(10000, 65535)
+def maybe_setup_loopback_network(request):
+    # unshare(1) -rn drops us in a new network namespace in which the "lo" is
+    # not up yet, so let's set it up first.
+    if request.config.getoption('--run-within-unshare'):
+        try:
+            args = "ip link set lo up".split()
+            subprocess.run(args, check=True)
+        except FileNotFoundError:
+            args = "/sbin/ifconfig lo up".split()
+            subprocess.run(args, check=True)
+    yield
 
-    server_process = subprocess.Popen([
-        sys.executable,
-        os.path.join(os.path.dirname(__file__), "rest_api_mock.py"),
-        ip,
-        str(port)])
 
+@pytest.fixture(scope="session")
+def rest_api_mock_server(request, maybe_setup_loopback_network):
+    ip = '127.0.0.1'
+    port = 12345
     server = (ip, port)
 
-    i = 0
-    while True:
+    server_process = subprocess.Popen([sys.executable,
+                                       os.path.join(os.path.dirname(__file__), "rest_api_mock.py"),
+                                       ip,
+                                       str(port)])
+    # wait 5 seconds for the expected requests
+    timeout = 5
+    interval = 0.1
+    for _ in range(int(timeout / interval)):
+        returncode = server_process.poll()
+        if returncode is not None:
+            # process terminated
+            raise subprocess.CalledProcessError(returncode, server_process.args)
         try:
             rest_api_mock.get_expected_requests(server)
             break
         except requests.exceptions.ConnectionError:
-            if i == 50:  # 5 seconds
-                raise
-            time.sleep(0.1)
-            i += 1
+            time.sleep(interval)
+    else:
+        server_process.terminate()
+        server_process.wait()
+        raise subprocess.TimeoutExpired(server_process.args, timeout)
 
     try:
         yield server

test/nodetool/suite.yaml

@@ -1,2 +1,3 @@
 type: Tool
 coverage: false
+launcher: unshare -rn pytest --run-within-unshare