Merge 'test/cql-pytest: run tests with tablets' from Botond Dénes

Add `--experimental-features=tablets` to both `test/cql-pytest/suite.yaml` and `test/cql-pytest/run.py`, so tablets are enabled. Detect tablet support in `contest.py` and add an xfail and skip marker to mark tests that fail/crash with tablets. These are expected to be fixed soon.

Some tests checking things around alter-keyspace, had to force-disable tablets on the created keyspace, because tablets interfere with the test (a keyspace with tablets cannot have simple strategy for example).
Tablets were also interfering with `test_keyspace.py:test_storage_options_local`, because it is expecting `system_schema.scylla_keyspaces` to not have any entries for local storage keyspace, but they have it if tablets are enabled. Adjust the test to account for this.

Closes scylladb/scylladb#16840

* github.com:scylladb/scylladb:
  test/cql-pytest: run.py,suite.yaml: enable tablets by default
  test/cql-pytest: sprinkle xfail_tablets and skip_with_tablets as needed
  test/cql-pytest: disable tablets for some keyspace-altering tests
  test/cql-pytest: test_keyspace.py: test_storage_options_local(): fix for tablets
  test/cql-pytest: fix test_tablets.py to set initial_tablets correctly
  test/cql-pytest: add tablet detection logic and fixtures
  test/cql-pytest: extract is_scylla check into util.py

test/cql-pytest/cassandra_tests/validation/operations/alter_test.py

@@ -204,10 +204,14 @@ def testCreateAlterKeyspaces(cql, test_keyspace, this_dc):
 # Test {@link ConfigurationException} thrown on alter keyspace to no DC
 # option in replication configuration.
 # Reproduces CASSANDRA-12681 and Scylla #10036
-def testAlterKeyspaceWithNoOptionThrowsConfigurationException(cql, test_keyspace, this_dc):
+def testAlterKeyspaceWithNoOptionThrowsConfigurationException(cql, test_keyspace, this_dc, has_tablets):
+    if has_tablets:
+        extra_opts = " AND TABLETS = {'enabled': false}"
+    else:
+        extra_opts = ""
     # Create keyspaces
-    with create_keyspace(cql, "replication={ 'class' : 'NetworkTopologyStrategy', '" + this_dc + "' : 3 }") as abc:
-        with create_keyspace(cql, "replication={ 'class' : 'NetworkTopologyStrategy', 'replication_factor' : 3 }") as xyz:
+    with create_keyspace(cql, "replication={ 'class' : 'NetworkTopologyStrategy', '" + this_dc + "' : 3 }" + extra_opts) as abc:
+        with create_keyspace(cql, "replication={ 'class' : 'NetworkTopologyStrategy', 'replication_factor' : 3 }" + extra_opts) as xyz:
             # Try to alter the created keyspace without any option
             assert_invalid_throw(cql, xyz, ConfigurationException, "ALTER KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy' }")
             assert_invalid_throw(cql, abc, ConfigurationException, "ALTER KEYSPACE %s WITH replication={ 'class' : 'NetworkTopologyStrategy' }")

test/cql-pytest/conftest.py

@@ -21,7 +21,7 @@ import tempfile
 import time
 import random
 
-from util import unique_name, new_test_table, cql_session, local_process_id
+from util import unique_name, new_test_keyspace, keyspace_has_tablets, cql_session, local_process_id, is_scylla
 
 
 print(f"Driver name {DRIVER_NAME}, version {DRIVER_VERSION}")
@@ -106,8 +106,7 @@ def test_keyspace(cql, this_dc):
 def scylla_only(cql):
     # We recognize Scylla by checking if there is any system table whose name
     # contains the word "scylla":
-    names = [row.table_name for row in cql.execute("SELECT * FROM system_schema.tables WHERE keyspace_name = 'system'")]
-    if not any('scylla' in name for name in names):
+    if not is_scylla(cql):
         pytest.skip('Scylla-only test skipped')
 
 # "cassandra_bug" is similar to "scylla_only", except instead of skipping
@@ -221,3 +220,18 @@ def temp_workdir():
     """ Creates a temporary work directory, for the scope of a single test. """
     with tempfile.TemporaryDirectory() as workdir:
         yield workdir
+
+@pytest.fixture(scope="session")
+def has_tablets(cql):
+    with new_test_keyspace(cql, " WITH REPLICATION = {'class' : 'NetworkTopologyStrategy', 'replication_factor': 1}") as keyspace:
+        return keyspace_has_tablets(cql, keyspace)
+
+@pytest.fixture(scope="function")
+def xfail_tablets(request, has_tablets):
+    if has_tablets:
+        request.node.add_marker(pytest.mark.xfail(reason='Test expected to fail with tablets experimental feature on'))
+
+@pytest.fixture(scope="function")
+def skip_with_tablets(has_tablets):
+    if has_tablets:
+        pytest.skip("Test may crash with tablets experimental feature on")

test/cql-pytest/run.py

@@ -311,6 +311,8 @@ def run_scylla_cmd(pid, dir):
         # test/alternator/run.
         '--experimental-features=udf',
         '--experimental-features=keyspace-storage-options',
+        '--experimental-features=consistent-topology-changes',
+        '--experimental-features=tablets',
         '--enable-user-defined-functions', '1',
         # Set up authentication in order to allow testing this module
         # and other modules dependent on it: e.g. service levels

test/cql-pytest/suite.yaml

@@ -2,3 +2,8 @@ type: Python
 pool_size: 4
 dirties_cluster:
   - test_native_transport
+extra_scylla_cmdline_options:
+  - '--experimental-features=udf'
+  - '--experimental-features=consistent-topology-changes'
+  - '--experimental-features=keyspace-storage-options'
+  - '--experimental-features=tablets'

test/cql-pytest/test_cdc.py

@@ -20,7 +20,8 @@ def wait_for_first_cdc_generation(cql, timeout):
         assert time.time() < deadline, "Timed out waiting for the first CDC generation"
         time.sleep(1)
 
-def test_cdc_log_entries_use_cdc_streams(scylla_only, cql, test_keyspace):
+# xfail_tablets due to https://github.com/scylladb/scylladb/issues/16317
+def test_cdc_log_entries_use_cdc_streams(scylla_only, cql, test_keyspace, xfail_tablets):
     '''Test that the stream IDs chosen for CDC log entries come from the CDC generation
     whose streams are listed in the streams description table. Since this test is executed
     on a single-node cluster, there is only one generation.'''
@@ -49,7 +50,8 @@ def test_cdc_log_entries_use_cdc_streams(scylla_only, cql, test_keyspace):
 
 # Test for #10473 - reading logs (from sstable) after dropping
 # column in base.
-def test_cdc_alter_table_drop_column(scylla_only, cql, test_keyspace):
+# xfail_tablets due to https://github.com/scylladb/scylladb/issues/16317
+def test_cdc_alter_table_drop_column(scylla_only, cql, test_keyspace, xfail_tablets):
     schema = "pk int primary key, v int"
     extra = " with cdc = {'enabled': true}"
     with new_test_table(cql, test_keyspace, schema, extra) as table:
@@ -62,7 +64,8 @@ def test_cdc_alter_table_drop_column(scylla_only, cql, test_keyspace):
 
 # Regression test for #12098 - check that LWT inserts don't observe
 # themselves inside preimages
-def test_cdc_with_lwt_preimage(scylla_only, cql, test_keyspace):
+# xfail_tablets due to https://github.com/scylladb/scylladb/issues/16317
+def test_cdc_with_lwt_preimage(scylla_only, cql, test_keyspace, xfail_tablets):
     schema = "pk int primary key"
     extra = " with cdc = {'enabled': true, 'preimage':true}"
     with new_test_table(cql, test_keyspace, schema, extra) as table:

test/cql-pytest/test_describe.py

@@ -389,7 +389,8 @@ def test_desc_schema(cql, test_keyspace, random_seed):
 
 # Test that `DESC CLUSTER` contains token ranges to endpoints map
 # The test is `scylla_only` because there is no `system.token_ring` table in Cassandra
-def test_desc_cluster(scylla_only, cql, test_keyspace):
+# xfail_tablets due to https://github.com/scylladb/scylladb/issues/16483
+def test_desc_cluster(scylla_only, cql, test_keyspace, xfail_tablets):
     cql.execute(f"USE {test_keyspace}")
     desc = cql.execute("DESC CLUSTER").one()
     desc_endpoints = []
@@ -534,7 +535,8 @@ def test_desc_udf_uda(cql, test_keyspace, scylla_only):
 # Example: caching = {'keys': 'ALL', 'rows_per_partition': 'ALL'}
 # Reproduces #14895
 # The test is marked scylla_only because it uses a Scylla-only property "cdc".
-def test_whitespaces_in_table_options(cql, test_keyspace, scylla_only):
+# xfail_tablets due to https://github.com/scylladb/scylladb/issues/16317
+def test_whitespaces_in_table_options(cql, test_keyspace, scylla_only, xfail_tablets):
     regex = "\\{[^}]*[:,][^\\s][^}]*\\}" # looks for any colon or comma without space after it inside a { }
     
     with new_test_table(cql, test_keyspace, "a int primary key", "WITH cdc = {'enabled': true}") as tbl:

test/cql-pytest/test_guardrail_replication_strategy.py

@@ -73,7 +73,7 @@ def test_given_non_empty_warn_and_fail_lists_when_creating_ks_should_fail_query_
                                                      warnings_count=0, fails_count=1)
 
 
-def test_given_already_existing_ks_when_altering_ks_should_validate_against_discouraged_strategies(cql, this_dc):
+def test_given_already_existing_ks_when_altering_ks_should_validate_against_discouraged_strategies(cql, this_dc, has_tablets):
     with ExitStack() as config_modifications:
         # place 1 strategy on warn list, 1 strategy on fail list and leave remaining strategies unspecified,
         # i.e. let them be allowed
@@ -82,8 +82,13 @@ def test_given_already_existing_ks_when_altering_ks_should_validate_against_disc
         config_modifications.enter_context(
             config_value_context(cql, 'replication_strategy_fail_list', 'EverywhereStrategy'))
 
+        if has_tablets:
+            extra_opts = " AND TABLETS = {'enabled': false}"
+        else:
+            extra_opts = ""
+
         # create a ks with "allowed" strategy
-        with new_test_keyspace(cql, " WITH REPLICATION = { 'class' : 'NetworkTopologyStrategy', '" + this_dc + "' : 3 }") as keyspace:
+        with new_test_keyspace(cql, " WITH REPLICATION = { 'class' : 'NetworkTopologyStrategy', '" + this_dc + "' : 3 }" + extra_opts) as keyspace:
             # alter this ks to use other strategy that is NOT present on any list
             response_future = cql.execute_async(
                 "ALTER KEYSPACE " + keyspace + " WITH REPLICATION = { 'class' : 'LocalStrategy', 'replication_factor' : 3 }")

test/cql-pytest/test_keyspace.py

@@ -112,8 +112,12 @@ def test_alter_keyspace_invalid(cql, this_dc):
 # replication_factor option. However, this is only true in Scylla - in
 # Cassandra 4.1 and above, a missing replication_factor *is* allowed,
 # because there is a default_keyspace_rf configuration. See issue #16028.
-def test_alter_keyspace_missing_rf(cql, this_dc, scylla_only):
-    with new_test_keyspace(cql, "WITH REPLICATION = { 'class' : 'NetworkTopologyStrategy', '" + this_dc + "' : 1 }") as keyspace:
+def test_alter_keyspace_missing_rf(cql, this_dc, scylla_only, has_tablets):
+    if has_tablets:
+        extra_opts = " AND TABLETS = {'enabled': false}"
+    else:
+        extra_opts = ""
+    with new_test_keyspace(cql, "WITH REPLICATION = { 'class' : 'NetworkTopologyStrategy', '" + this_dc + "' : 1 }" + extra_opts) as keyspace:
         # SimpleStrategy, if not outright forbidden, requires a
         # replication_factor option.
         with pytest.raises(ConfigurationException):
@@ -207,9 +211,15 @@ def test_concurrent_create_and_drop_keyspace(cql, this_dc, fails_without_consist
 def test_storage_options_local(cql, scylla_only):
     ksdef = "WITH REPLICATION = { 'class' : 'NetworkTopologyStrategy', 'replication_factor' : '1' } " \
             "AND STORAGE = { 'type' : 'LOCAL' }"
+
+    def row_has_storage_options(row):
+        o = getattr(row, 'storage_options', None)
+        t = getattr(row, 'storage_type', None)
+        return t is not None or o is not None
+
     with new_test_keyspace(cql, ksdef) as keyspace:
-        res = cql.execute(f"SELECT * FROM system_schema.scylla_keyspaces WHERE keyspace_name = '{keyspace}'")
-        assert not res.all()
+        res = list(cql.execute(f"SELECT * FROM system_schema.scylla_keyspaces WHERE keyspace_name = '{keyspace}'"))
+        assert not res or not row_has_storage_options(res[0])
 
 # Test that passing an unsupported storage type is not legal
 def test_storage_options_unknown_type(cql, scylla_only):

test/cql-pytest/test_select_from_mutation_fragments.py

@@ -27,7 +27,8 @@ def test_table(cql, test_keyspace):
         yield table
 
 
-def test_smoke(cql, test_table, scylla_only):
+# skip_with_tablets due to https://github.com/scylladb/scylladb/issues/16484
+def test_smoke(cql, test_table, scylla_only, skip_with_tablets):
     """ Simple smoke tests, this should fail first if something is very wrong. """
     partitions = {}
     for i in range(0, 1):
@@ -157,7 +158,8 @@ def test_count(cql, test_table, scylla_only):
     check_count('partition end', 1)
 
 
-def test_many_partition_scan(cql, test_keyspace, scylla_only):
+# skip_with_tablets due to https://github.com/scylladb/scylladb/issues/16484
+def test_many_partition_scan(cql, test_keyspace, scylla_only, skip_with_tablets):
     """
     Full scans work like secondary-index based scans. First, a query is
     issued to obtain partition-keys, then each partition is read individually.
@@ -199,7 +201,8 @@ def test_many_partition_scan(cql, test_keyspace, scylla_only):
         assert actual_partitions == partitions
 
 
-def test_metadata_and_value(cql, test_keyspace, scylla_path, scylla_data_dir, scylla_only):
+# skip_with_tablets due to https://github.com/scylladb/scylladb/issues/16484
+def test_metadata_and_value(cql, test_keyspace, scylla_path, scylla_data_dir, scylla_only, skip_with_tablets):
     """
     Test that metadata + value columns allow reconstructing a full sstable dump.
     Meaning that their json representation of metadata and value is the same.
@@ -424,7 +427,8 @@ def test_ck_in_query(cql, test_table):
             assert getattr(row, col_name) == expected_value
 
 
-def test_many_partitions(cql, test_keyspace, scylla_only):
+# skip_with_tablets due to https://github.com/scylladb/scylladb/issues/16484
+def test_many_partitions(cql, test_keyspace, scylla_only, skip_with_tablets):
     num_partitions = 5000
     with util.new_test_table(cql, test_keyspace, 'pk int PRIMARY KEY, v int') as table:
         delete_id = cql.prepare(f"DELETE FROM {table} WHERE pk = ?")

test/cql-pytest/test_tablets.py

@@ -25,7 +25,7 @@ from cassandra.protocol import ConfigurationException
 def test_keyspace_128_tablets(cql, this_dc):
     name = unique_name()
     try:
-        cql.execute("CREATE KEYSPACE " + name + " WITH REPLICATION = { 'class' : 'NetworkTopologyStrategy', '" + this_dc + "': 1, 'initial_tablets': 128 }")
+        cql.execute("CREATE KEYSPACE " + name + " WITH REPLICATION = { 'class' : 'NetworkTopologyStrategy', '" + this_dc + "': 1 } AND TABLETS = { 'enabled': true, 'initial': 128 }")
     except ConfigurationException:
         pytest.skip('Scylla does not support initial_tablets, or the tablets feature is not enabled')
     yield name

test/cql-pytest/test_tombstone_limit.py

@@ -247,7 +247,8 @@ def check_pages_many_partitions(results, expected):
         assert p == expected_pages.get(i, [])
 
 
-def test_partition_tombstone_prefix(cql, test_keyspace, lowered_tombstone_limit, driver_bug_1):
+# xfail_tablets due to https://github.com/scylladb/scylladb/issues/16486
+def test_partition_tombstone_prefix(cql, test_keyspace, lowered_tombstone_limit, driver_bug_1, xfail_tablets):
     with new_test_table(cql, test_keyspace, 'pk int, ck int, v int, PRIMARY KEY (pk, ck)') as table:
         insert_row_id = cql.prepare(f"INSERT INTO {table} (pk, ck, v) VALUES (?, ?, ?)")
         delete_partition_id = cql.prepare(f"DELETE FROM {table} WHERE pk = ?")
@@ -267,8 +268,8 @@ def test_partition_tombstone_prefix(cql, test_keyspace, lowered_tombstone_limit,
         check_pages_many_partitions(cql.execute(statement), {-1: all_pks[-1]})
 
 
-
-def test_partition_tombstone_span(cql, test_keyspace, lowered_tombstone_limit, driver_bug_1):
+# xfail_tablets due to https://github.com/scylladb/scylladb/issues/16486
+def test_partition_tombstone_span(cql, test_keyspace, lowered_tombstone_limit, driver_bug_1, xfail_tablets):
     with new_test_table(cql, test_keyspace, 'pk int, ck int, v int, PRIMARY KEY (pk, ck)') as table:
         insert_row_id = cql.prepare(f"INSERT INTO {table} (pk, ck, v) VALUES (?, ?, ?)")
         delete_partition_id = cql.prepare(f"DELETE FROM {table} WHERE pk = ?")
@@ -288,7 +289,8 @@ def test_partition_tombstone_span(cql, test_keyspace, lowered_tombstone_limit, d
         check_pages_many_partitions(cql.execute(statement), {0: all_pks[0], -1: all_pks[-1]})
 
 
-def test_static_row_tombstone_prefix(cql, test_keyspace, lowered_tombstone_limit, driver_bug_1):
+# xfail_tablets due to https://github.com/scylladb/scylladb/issues/16486
+def test_static_row_tombstone_prefix(cql, test_keyspace, lowered_tombstone_limit, driver_bug_1, xfail_tablets):
     with new_test_table(cql, test_keyspace, 'pk int, ck int, v int, s int static, PRIMARY KEY (pk, ck)') as table:
         upsert_row_id = cql.prepare(f"UPDATE {table} SET s = ? WHERE pk = ?")
         delete_partition_id = cql.prepare(f"DELETE FROM {table} WHERE pk = ?")
@@ -308,7 +310,8 @@ def test_static_row_tombstone_prefix(cql, test_keyspace, lowered_tombstone_limit
         check_pages_many_partitions(cql.execute(statement), {-1: all_pks[-1]})
 
 
-def test_static_row_tombstone_span(cql, test_keyspace, lowered_tombstone_limit, driver_bug_1):
+# xfail_tablets due to https://github.com/scylladb/scylladb/issues/16486
+def test_static_row_tombstone_span(cql, test_keyspace, lowered_tombstone_limit, driver_bug_1, xfail_tablets):
     with new_test_table(cql, test_keyspace, 'pk int, ck int, v int, s int static, PRIMARY KEY (pk, ck)') as table:
         upsert_row_id = cql.prepare(f"UPDATE {table} SET s = ? WHERE pk = ?")
         delete_partition_id = cql.prepare(f"DELETE FROM {table} WHERE pk = ?")

test/cql-pytest/util.py

@@ -63,6 +63,39 @@ def format_tuples(tuples=None, **kwargs):
     body = ', '.join(f"'{key}': '{value}'" for key, value in tuples.items())
     return f'{{ {body} }}'
 
+def is_scylla(cql):
+    """ Check whether we are running against Scylla or not """
+    # We recognize Scylla by checking if there is any system table whose name
+    # contains the word "scylla":
+    names = [row.table_name for row in cql.execute("SELECT * FROM system_schema.tables WHERE keyspace_name = 'system'")]
+    return any('scylla' in name for name in names)
+
+def keyspace_has_tablets(cql, keyspace):
+    """ Return true if the keyspace was created with tablets.
+
+    We support running cql-pytest against an older version of scylla, so we do
+    the detection in a way that accounts for scylla possibly not even knowing
+    what tablets is.
+    For cassandra, this will always return no.
+
+    If the keyspace was created with tablets, it will have an entry in
+    `system_schema.scylla_keyspaces`, with `initial_tablets` set.
+    So here, we simply query this table, looking for a partition for the
+    appropriate keyspace. If the result has the `initial_tablets` column and it
+    is set, the keyspace has tablets.
+    """
+    if not is_scylla(cql):
+        return False
+
+    # Need to use network strategy, otherwise tablets will not be enabled.
+    res = list(cql.execute(f"SELECT * FROM system_schema.scylla_keyspaces WHERE keyspace_name='{keyspace}'"))
+    # The row migh exist due to storage related options, but the tablets related fields are null.
+    # So we check that:
+    # * the row exists
+    # * `initial_tablets` has a value
+    if not res:
+        return False
+    return getattr(res[0], "initial_tablets", None) is not None
 
 # A utility function for creating a new temporary keyspace with given options.
 # It can be used in a "with", as: