bahman/scylla
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

vector_search: Restrict vector index tests to tablets only

Browse Source 
Vector indexes are going to be supported only for tablets (see VECTOR-322).
As a result, tests using vector indexes will be failing when run with vnodes.

This change ensures tests using vector indexes run exclusively with tablets.

Fixes: VECTOR-49

Closes scylladb/scylladb#27233
This commit is contained in:
Karol Nowacki
2025-11-25 14:02:19 +01:00
committed by Nadav Har'El
parent c75375b148
commit 3edd12eba0
7 changed files with 189 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
test/boost/cdc_test.cc
View File

@@ -47,6 +47,16 @@ api::timestamp_type find_timestamp(const mutation&);
utils::UUID generate_timeuuid(api::timestamp_type);
}
namespace {
cql_test_config enable_tablets() {
cql_test_config cfg;
cfg.initial_tablets = 1;
return cfg;
}
} // namespace
BOOST_AUTO_TEST_SUITE(cdc_test)
SEASTAR_THREAD_TEST_CASE(test_find_mutation_timestamp) {
@@ -199,62 +209,73 @@ SEASTAR_THREAD_TEST_CASE(test_detecting_conflict_of_cdc_log_table_with_existing_
BOOST_REQUIRE_THROW(e.execute_cql("CREATE TABLE ks.tbl (a int PRIMARY KEY) WITH cdc = {'enabled': true}").get(), exceptions::invalid_request_exception);
BOOST_REQUIRE(!e.local_db().has_schema("ks", "tbl"));
e.execute_cql("CREATE TABLE ks.tbl (a int PRIMARY KEY, b vector<float, 3>)").get();
e.execute_cql("CREATE TABLE ks.tbl (a int PRIMARY KEY)").get();
BOOST_REQUIRE(e.local_db().has_schema("ks", "tbl"));
// Conflict on ALTER which enables cdc log
BOOST_REQUIRE_THROW(e.execute_cql("ALTER TABLE ks.tbl WITH cdc = {'enabled': true}").get(), exceptions::invalid_request_exception);
// Conflict on CREATE INDEX which enables cdc log for vector search
BOOST_REQUIRE_THROW(e.execute_cql("CREATE INDEX ON ks.tbl (b) USING 'vector_index'").get(), exceptions::invalid_request_exception);
}).get();
}
SEASTAR_THREAD_TEST_CASE(test_permissions_of_cdc_log_table) {
SEASTAR_THREAD_TEST_CASE(test_detecting_conflict_of_cdc_log_table_with_existing_table_when_vector_index_created) {
do_with_cql_env_thread([] (cql_test_env& e) {
auto assert_unauthorized = [&e] (const sstring& stmt) {
testlog.info("Must throw unauthorized_exception: {}", stmt);
BOOST_REQUIRE_THROW(e.execute_cql(stmt).get(), exceptions::unauthorized_exception);
};
// Conflict on CREATE which enables cdc log
e.execute_cql("CREATE TABLE ks.tbl_scylla_cdc_log (a int PRIMARY KEY)").get();
// Allow MODIFY, SELECT, ALTER
auto log_table = "ks." + cdc::log_name("tbl");
auto stream_id = cdc::log_meta_column_name("stream_id");
auto time = cdc::log_meta_column_name("time");
auto batch_seq_no = cdc::log_meta_column_name("batch_seq_no");
auto ttl = cdc::log_meta_column_name("ttl");
e.execute_cql("CREATE TABLE ks.tbl (a int PRIMARY KEY, b vector<float, 3>)").get();
BOOST_REQUIRE(e.local_db().has_schema("ks", "tbl"));
auto cdc_enablement_queries = {
"ALTER TABLE ks.tbl WITH cdc = {'enabled': true}",
"CREATE INDEX ON ks.tbl (b) USING 'vector_index'",
};
// Conflict on CREATE INDEX which enables cdc log for vector search
BOOST_REQUIRE_THROW(e.execute_cql("CREATE INDEX ON ks.tbl (b) USING 'vector_index'").get(), exceptions::invalid_request_exception);
}, enable_tablets()).get();
}
for (auto& q : cdc_enablement_queries) {
e.execute_cql("CREATE TABLE ks.tbl (a int PRIMARY KEY, b vector<float, 3>)").get();
BOOST_REQUIRE(e.local_db().has_schema("ks", "tbl"));
SEASTAR_THREAD_TEST_CASE(test_permissions_of_cdc_log_table) {
e.execute_cql(q).get();
BOOST_REQUIRE(e.local_db().has_schema("ks", cdc::log_name("tbl")));
std::vector<std::pair<sstring, cql_test_config>> cdc_enablement_queries = {
{"ALTER TABLE ks.tbl WITH cdc = {'enabled': true}", cql_test_config{}},
{"CREATE INDEX ON ks.tbl (b) USING 'vector_index'", enable_tablets()},
};
e.execute_cql(format("INSERT INTO {} (\"{}\", \"{}\", \"{}\") VALUES (0x00000000000000000000000000000000, now(), 0)",
log_table, stream_id, time, batch_seq_no
)).get();
e.execute_cql(format("UPDATE {} SET \"{}\"= 100 WHERE \"{}\" = 0x00000000000000000000000000000000 AND \"{}\" = now() AND \"{}\" = 0",
log_table, ttl, stream_id, time, batch_seq_no
)).get();
e.execute_cql(format("DELETE FROM {} WHERE \"{}\" = 0x00000000000000000000000000000000 AND \"{}\" = now() AND \"{}\" = 0",
log_table, stream_id, time, batch_seq_no
)).get();
e.execute_cql("SELECT * FROM " + log_table).get();
e.execute_cql("ALTER TABLE " + log_table + " WITH comment = 'some not very interesting comment'").get();
for (auto& [q, cfg] : cdc_enablement_queries) {
do_with_cql_env_thread([&] (cql_test_env& e) {
auto assert_unauthorized = [&e] (const sstring& stmt) {
testlog.info("Must throw unauthorized_exception: {}", stmt);
BOOST_REQUIRE_THROW(e.execute_cql(stmt).get(), exceptions::unauthorized_exception);
};
// Disallow DROP
assert_unauthorized("DROP TABLE " + log_table);
// Allow MODIFY, SELECT, ALTER
auto log_table = "ks." + cdc::log_name("tbl");
auto stream_id = cdc::log_meta_column_name("stream_id");
auto time = cdc::log_meta_column_name("time");
auto batch_seq_no = cdc::log_meta_column_name("batch_seq_no");
auto ttl = cdc::log_meta_column_name("ttl");
e.execute_cql("DROP TABLE ks.tbl").get();
}
}).get();
e.execute_cql("CREATE TABLE ks.tbl (a int PRIMARY KEY, b vector<float, 3>)").get();
BOOST_REQUIRE(e.local_db().has_schema("ks", "tbl"));
e.execute_cql(q).get();
BOOST_REQUIRE(e.local_db().has_schema("ks", cdc::log_name("tbl")));
e.execute_cql(format("INSERT INTO {} (\"{}\", \"{}\", \"{}\") VALUES (0x00000000000000000000000000000000, now(), 0)",
log_table, stream_id, time, batch_seq_no
)).get();
e.execute_cql(format("UPDATE {} SET \"{}\"= 100 WHERE \"{}\" = 0x00000000000000000000000000000000 AND \"{}\" = now() AND \"{}\" = 0",
log_table, ttl, stream_id, time, batch_seq_no
)).get();
e.execute_cql(format("DELETE FROM {} WHERE \"{}\" = 0x00000000000000000000000000000000 AND \"{}\" = now() AND \"{}\" = 0",
log_table, stream_id, time, batch_seq_no
)).get();
e.execute_cql("SELECT * FROM " + log_table).get();
e.execute_cql("ALTER TABLE " + log_table + " WITH comment = 'some not very interesting comment'").get();
// Disallow DROP
assert_unauthorized("DROP TABLE " + log_table);
e.execute_cql("DROP TABLE ks.tbl").get();
}, cfg).get();
}
}
SEASTAR_THREAD_TEST_CASE(test_disallow_cdc_on_materialized_view) {
@@ -312,18 +333,17 @@ SEASTAR_THREAD_TEST_CASE(test_permissions_of_cdc_description) {
}
SEASTAR_THREAD_TEST_CASE(test_cdc_log_schema) {
do_with_cql_env_thread([] (cql_test_env& e) {
int required_column_count = 0;
const auto base_tbl_name = "tbl";
std::vector<std::pair<sstring, cql_test_config>> cdc_enablement_queries = {
{format("ALTER TABLE {} WITH cdc = {{'enabled': true}}", base_tbl_name), cql_test_config{}},
{format("CREATE INDEX ON {} (c_vec) USING 'vector_index'", base_tbl_name), enable_tablets()},
};
for (auto& [q, cfg] : cdc_enablement_queries) {
do_with_cql_env_thread([&] (cql_test_env& e) {
int required_column_count = 0;
const auto base_tbl_name = "tbl";
e.execute_cql(format("CREATE TYPE {} (x int)", "typ")).get();
e.execute_cql(format("CREATE TYPE {} (x int)", "typ")).get();
auto cdc_enablement_queries = {
format("ALTER TABLE {} WITH cdc = {{'enabled': true}}", base_tbl_name),
format("CREATE INDEX ON {} (c_vec) USING 'vector_index'", base_tbl_name),
};
for (auto& q : cdc_enablement_queries) {
e.execute_cql(format("CREATE TABLE {} (pk int, ck int, s int static, c int, "
"c_list list<int>, c_map map<int, int>, c_set set<int>, c_typ typ, c_vec vector<float, 3>,"
"PRIMARY KEY (pk, ck))", base_tbl_name)).get();
@@ -410,8 +430,9 @@ SEASTAR_THREAD_TEST_CASE(test_cdc_log_schema) {
e.execute_cql("DROP TABLE ks.tbl").get();
required_column_count = 0;
}
}).get();
}, cfg).get();
}
}
} // cdc_test namespace

7
test/boost/cql_auth_query_test.cc
View File

@@ -47,6 +47,11 @@ static shared_ptr<db::config> db_config_with_auth() {
return config_ptr;
}
static cql_test_config enable_tablets(cql_test_config cfg) {
cfg.initial_tablets = 1;
return cfg;
}
//
// These functions must be called inside Seastar threads.
//
@@ -379,7 +384,7 @@ SEASTAR_TEST_CASE(select_from_vector_indexed_table) {
exception_predicate::message_contains("User bob has no SELECT permission"));
});
},
db_config_with_auth());
enable_tablets(db_config_with_auth()));
}
BOOST_AUTO_TEST_SUITE_END()

3
test/cql/vs_cdc_enable_disable_test.cql
View File

@@ -2,10 +2,9 @@
-- do not depend on how stream IDs are assigned to partition keys.
-- Error messages contain a keyspace name. Make the output stable.
-- CDC and tablets are not working together yet, turn them off.
CREATE KEYSPACE ks
WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': 1} AND
tablets = {'enabled': false};
tablets = {'enabled': true};
-- create table with vector column
create table ks.t (pk int, ck int, v vector<float, 3>, primary key(pk, ck));

3
test/cql/vs_cdc_enable_disable_test.result
View File

@@ -2,10 +2,9 @@
> -- do not depend on how stream IDs are assigned to partition keys.
>
> -- Error messages contain a keyspace name. Make the output stable.
> -- CDC and tablets are not working together yet, turn them off.
> CREATE KEYSPACE ks
> WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': 1} AND
> tablets = {'enabled': false};
> tablets = {'enabled': true};
OK
>
> -- create table with vector column

76
test/cqlpy/test_describe.py
View File

@@ -60,6 +60,9 @@ def extract_names(desc_result_iter: Iterable[DescRowType]) -> Iterable[str]:
def extract_create_statements(desc_result_iter: Iterable[DescRowType]) -> Iterable[str]:
return map(lambda result: result.create_statement, desc_result_iter)
def is_create_index(create_statement: str) -> bool:
return create_statement.strip().upper().startswith("CREATE INDEX")
# (`element` refers to keyspace or keyspace's element(table, type, function, aggregate))
# There are 2 main types of tests:
# - tests for listings (DESC TABLES/DESC KEYSPACES/DESC TYPES/...)
@@ -238,10 +241,7 @@ def test_desc_table(cql, test_keyspace, random_seed, has_tablets):
# This test compares the content of `system_schema.tables` and `system_schema.columns` tables
# when providing tablet options to CREATE TABLE.
def test_desc_table_with_tablet_options(cql, test_keyspace, random_seed, has_tablets):
if has_tablets: # issue #18180
global counter_table_chance
counter_table_chance = 0
def test_desc_table_with_tablet_options(cql, test_keyspace, random_seed, skip_without_tablets):
tablet_options = {
'min_tablet_count': '100',
'min_per_shard_tablet_count': '0.8', # Verify that a floating point value works for this hint
@@ -992,7 +992,10 @@ def test_table_options_quoting(cql, test_keyspace):
["ALTER TABLE {t} WITH cdc = {{'enabled': true}}",
"CREATE INDEX ON {t}(b) USING 'vector_index'"],
ids=["alter", "create_index"])
def test_hide_cdc_table(scylla_only, cql, test_keyspace, cdc_enablement_query):
def test_hide_cdc_table(scylla_only, cql, test_keyspace, cdc_enablement_query, has_tablets):
if is_create_index(cdc_enablement_query) and not has_tablets:
pytest.skip("Test needs tablets experimental feature on")
cdc_table_suffix = "_scylla_cdc_log"
with new_test_table(cql, test_keyspace, "a int primary key, b vector<float, 3>") as t:
t_name = t.split('.')[1]
@@ -1042,7 +1045,9 @@ def test_hide_cdc_table(scylla_only, cql, test_keyspace, cdc_enablement_query):
["ALTER TABLE {t} WITH cdc = {{'enabled': true}}",
"CREATE INDEX ON {t}(v) USING 'vector_index'"],
ids=["alter", "create_index"])
def test_describe_cdc_log_table_format(scylla_only, cql, test_keyspace, cdc_enablement_query):
def test_describe_cdc_log_table_format(scylla_only, cql, test_keyspace, cdc_enablement_query, has_tablets):
if is_create_index(cdc_enablement_query) and not has_tablets:
pytest.skip("Test needs tablets experimental feature on")
with new_test_table(cql, test_keyspace, "p int PRIMARY KEY, v vector<float, 3>") as table:
log_table = f"{table}_scylla_cdc_log"
_, log_table_name = log_table.split(".")
@@ -1070,7 +1075,10 @@ def test_describe_cdc_log_table_format(scylla_only, cql, test_keyspace, cdc_enab
["ALTER TABLE {t} WITH cdc = {{'enabled': true}}",
"CREATE INDEX ON {t}(v) USING 'vector_index'"],
ids=["alter", "create_index"])
def test_describe_cdc_log_table_create_statement(scylla_only, cql, test_keyspace, cdc_enablement_query):
def test_describe_cdc_log_table_create_statement(scylla_only, cql, test_keyspace, cdc_enablement_query, has_tablets):
if is_create_index(cdc_enablement_query) and not has_tablets:
pytest.skip("Test needs tablets experimental feature on")
def format_create_statement(stmt: str) -> str:
stmt = " ".join(stmt.split("\n"))
stmt = " ".join(stmt.split())
@@ -1131,7 +1139,10 @@ def test_describe_cdc_log_table_create_statement(scylla_only, cql, test_keyspace
["ALTER TABLE {t} WITH cdc = {{'enabled': true}}",
"CREATE INDEX ON {t}(v) USING 'vector_index'"],
ids=["alter", "create_index"])
def test_describe_cdc_log_table_opts(scylla_only, cql, test_keyspace, cdc_enablement_query):
def test_describe_cdc_log_table_opts(scylla_only, cql, test_keyspace, cdc_enablement_query, has_tablets):
if is_create_index(cdc_enablement_query) and not has_tablets:
pytest.skip("Test needs tablets experimental feature on")
def test_config(altered_cdc_log_table_opt):
with new_test_table(cql, test_keyspace, "p int PRIMARY KEY, v vector<float, 3>") as table:
log_table = f"{table}_scylla_cdc_log"
@@ -1608,7 +1619,8 @@ class AuthSLContext:
def __enter__(self):
if self.ks:
self.cql.execute(f"CREATE KEYSPACE {self.ks} WITH REPLICATION = {{ 'class': 'SimpleStrategy', 'replication_factor': 1 }}")
self.cql.execute(f"CREATE KEYSPACE {self.ks} WITH REPLICATION = {{ 'class': 'NetworkTopologyStrategy', 'replication_factor': 1 }}")
self.driver_sl = self.cql.execute("LIST SERVICE LEVEL driver").one()
return self
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
@@ -2884,7 +2896,6 @@ def test_desc_restore(cql):
cql.execute(f"CREATE TYPE {ks}.my_type (value int)")
cql.execute(f"""CREATE TABLE {ks}.some_other_table (c1 frozen<my_type>, c2 double, c3 int, c4 set<int>,
PRIMARY KEY ((c1, c2), c3)) WITH comment = 'some comment'""")
cql.execute(f"CREATE TABLE {ks}.vector_table (pk int PRIMARY KEY, v vector<float, 3>)")
cql.execute(f"""CREATE MATERIALIZED VIEW {ks}.mv AS
SELECT pk FROM {ks}.my_table
@@ -2894,10 +2905,6 @@ def test_desc_restore(cql):
cql.execute(f"CREATE INDEX myindex ON {ks}.some_other_table (c1)")
# Scylla doesn't support `sai`, while Cassandra doesn't support `vector_index`.
vec_class = "vector_index" if is_scylla(cql) else "sai"
cql.execute(f"CREATE INDEX custom_index ON {ks}.vector_table (v) USING '{vec_class}'")
# Scylla supports UDFs with Lua. Cassandra supports UDFs with Java.
if is_scylla(cql):
cql.execute(f"""CREATE FUNCTION {ks}.my_udf(val1 int, val2 int)
@@ -2973,6 +2980,47 @@ def test_desc_restore(cql):
finally:
cql.execute(f"DROP KEYSPACE IF EXISTS {ks}")
def test_desc_restore_vector_index(cql, skip_without_tablets):
"""
Verify that restoring the schema for vector index works correctly.
"""
restore_stmts = None
ks = "my_ks_vector"
with AuthSLContext(cql, ks=ks):
cql.execute(f"CREATE TABLE {ks}.vector_table (pk int PRIMARY KEY, v vector<float, 3>)")
# Scylla doesn't support `sai`, while Cassandra doesn't support `vector_index`.
vec_class = "vector_index" if is_scylla(cql) else "sai"
cql.execute(f"CREATE INDEX custom_index ON {ks}.vector_table (v) USING '{vec_class}'")
if is_scylla(cql):
restore_stmts = list(cql.execute("DESC SCHEMA WITH INTERNALS AND PASSWORDS"))
else:
restore_stmts = list(cql.execute("DESC SCHEMA WITH INTERNALS"))
def remove_other_keyspaces(rows: Iterable[DescRowType]) -> Iterable[DescRowType]:
return filter(lambda row: row.keyspace_name == ks or row.keyspace_name == None, rows)
restore_stmts = list(remove_other_keyspaces(restore_stmts))
with AuthSLContext(cql):
try:
for stmt in extract_create_statements(restore_stmts):
cql.execute(stmt)
if is_scylla(cql):
res = list(cql.execute("DESC SCHEMA WITH INTERNALS AND PASSWORDS"))
else:
res = list(cql.execute("DESC SCHEMA WITH INTERNALS"))
res = list(remove_other_keyspaces(res))
assert restore_stmts == res
finally:
cql.execute(f"DROP KEYSPACE IF EXISTS {ks}")
### ===========================================================================
# Verifies that describing a UDF with built-in types works correctly.

41
test/cqlpy/test_vector_index.py
View File

@@ -10,14 +10,13 @@ import pytest
from .util import new_test_table, is_scylla, unique_name
from cassandra.protocol import InvalidRequest, ConfigurationException
@pytest.mark.parametrize("test_keyspace", ["tablets", "vnodes"], indirect=True)
def test_create_vector_search_index(cql, test_keyspace, scylla_only):
def test_create_vector_search_index(cql, test_keyspace, scylla_only, skip_without_tablets):
schema = 'p int primary key, v vector<float, 3>'
with new_test_table(cql, test_keyspace, schema) as table:
cql.execute(f"CREATE CUSTOM INDEX ON {table}(v) USING 'vector_index'")
def test_create_vector_search_index_without_custom_keyword(cql, test_keyspace):
def test_create_vector_search_index_without_custom_keyword(cql, test_keyspace, skip_without_tablets):
schema = 'p int primary key, v vector<float, 3>'
with new_test_table(cql, test_keyspace, schema) as table:
if is_scylla(cql):
@@ -40,19 +39,19 @@ def test_create_custom_index_without_custom_class(cql, test_keyspace):
with pytest.raises((InvalidRequest, ConfigurationException), match=r"CUSTOM index requires specifying|Unable to find"):
cql.execute(f"CREATE CUSTOM INDEX ON {table}(v)")
def test_create_vector_search_index_on_nonvector_column(cql, test_keyspace, scylla_only):
def test_create_vector_search_index_on_nonvector_column(cql, test_keyspace, scylla_only, skip_without_tablets):
schema = 'p int primary key, v int'
with new_test_table(cql, test_keyspace, schema) as table:
with pytest.raises(InvalidRequest, match="Vector indexes are only supported on columns of vectors of floats"):
cql.execute(f"CREATE CUSTOM INDEX ON {table}(v) USING 'vector_index'")
def test_create_vector_search_index_with_bad_options(cql, test_keyspace, scylla_only):
def test_create_vector_search_index_with_bad_options(cql, test_keyspace, scylla_only, skip_without_tablets):
schema = 'p int primary key, v vector<float, 3>'
with new_test_table(cql, test_keyspace, schema) as table:
with pytest.raises(InvalidRequest, match="Unsupported option"):
cql.execute(f"CREATE CUSTOM INDEX ON {table}(v) USING 'vector_index' WITH OPTIONS = {{'bad_option': 'bad_value'}}")
def test_create_vector_search_index_with_bad_numeric_value(cql, test_keyspace, scylla_only):
def test_create_vector_search_index_with_bad_numeric_value(cql, test_keyspace, scylla_only, skip_without_tablets):
schema = 'p int primary key, v vector<float, 3>'
with new_test_table(cql, test_keyspace, schema) as table:
for val in ['-1', '513']:
@@ -64,19 +63,19 @@ def test_create_vector_search_index_with_bad_numeric_value(cql, test_keyspace, s
with pytest.raises(InvalidRequest, match="out of valid range"):
cql.execute(f"CREATE CUSTOM INDEX ON {table}(v) USING 'vector_index' WITH OPTIONS = {{'construction_beam_width': '5000' }}")
def test_create_vector_search_index_with_bad_similarity_value(cql, test_keyspace, scylla_only):
def test_create_vector_search_index_with_bad_similarity_value(cql, test_keyspace, scylla_only, skip_without_tablets):
schema = 'p int primary key, v vector<float, 3>'
with new_test_table(cql, test_keyspace, schema) as table:
with pytest.raises(InvalidRequest, match="Unsupported similarity function"):
cql.execute(f"CREATE CUSTOM INDEX ON {table}(v) USING 'vector_index' WITH OPTIONS = {{'similarity_function': 'bad_similarity_function'}}")
def test_create_vector_search_index_on_nonfloat_vector_column(cql, test_keyspace, scylla_only):
def test_create_vector_search_index_on_nonfloat_vector_column(cql, test_keyspace, scylla_only, skip_without_tablets):
schema = 'p int primary key, v vector<int, 3>'
with new_test_table(cql, test_keyspace, schema) as table:
with pytest.raises(InvalidRequest, match="Vector indexes are only supported on columns of vectors of floats"):
cql.execute(f"CREATE CUSTOM INDEX ON {table}(v) USING 'vector_index'")
def test_no_view_for_vector_search_index(cql, test_keyspace, scylla_only):
def test_no_view_for_vector_search_index(cql, test_keyspace, scylla_only, skip_without_tablets):
schema = 'p int primary key, v vector<float, 3>'
with new_test_table(cql, test_keyspace, schema) as table:
cql.execute(f"CREATE CUSTOM INDEX abc ON {table}(v) USING 'vector_index'")
@@ -87,7 +86,7 @@ def test_no_view_for_vector_search_index(cql, test_keyspace, scylla_only):
result = cql.execute(f"SELECT * FROM system_schema.views WHERE keyspace_name = '{test_keyspace}' AND view_name = 'def_index'")
assert len(result.current_rows) == 1, "Regular index should create a view in system_schema.views"
def test_describe_custom_index(cql, test_keyspace):
def test_describe_custom_index(cql, test_keyspace, skip_without_tablets):
schema = 'p int primary key, v1 vector<float, 3>, v2 vector<float, 3>'
with new_test_table(cql, test_keyspace, schema) as table:
# Cassandra inserts a space between the table name and parentheses,
@@ -115,7 +114,7 @@ def test_describe_custom_index(cql, test_keyspace):
assert f"CREATE CUSTOM INDEX custom1 ON {table}{maybe_space}(v2) USING '{custom_class}'" in b_desc
def test_vector_index_version_on_recreate(cql, test_keyspace, scylla_only):
def test_vector_index_version_on_recreate(cql, test_keyspace, scylla_only, skip_without_tablets):
schema = 'p int primary key, v vector<float, 3>'
with new_test_table(cql, test_keyspace, schema) as table:
_, table_name = table.split('.')
@@ -144,7 +143,7 @@ def test_vector_index_version_on_recreate(cql, test_keyspace, scylla_only):
assert result.current_rows[0].options['index_version'] != version
def test_vector_index_version_unaffected_by_alter(cql, test_keyspace, scylla_only):
def test_vector_index_version_unaffected_by_alter(cql, test_keyspace, scylla_only, skip_without_tablets):
schema = 'p int primary key, v vector<float, 3>'
with new_test_table(cql, test_keyspace, schema) as table:
_, table_name = table.split('.')
@@ -179,7 +178,7 @@ def test_vector_index_version_fail_given_as_option(cql, test_keyspace, scylla_on
with pytest.raises(InvalidRequest, match="Cannot specify index_version as a CUSTOM option"):
cql.execute(f"CREATE CUSTOM INDEX abc ON {table}(v) USING 'vector_index' WITH OPTIONS = {{'index_version': '18ad2003-05ea-17d9-1855-0325ac0a755d'}}")
def test_one_vector_index_on_column(cql, test_keyspace):
def test_one_vector_index_on_column(cql, test_keyspace, skip_without_tablets):
schema = "p int primary key, v vector<float, 3>"
if is_scylla(cql):
custom_class = 'vector_index'
@@ -192,7 +191,7 @@ def test_one_vector_index_on_column(cql, test_keyspace):
cql.execute(f"CREATE CUSTOM INDEX IF NOT EXISTS ON {table}(v) USING '{custom_class}'")
# Reproduces issue #26672
def test_two_same_name_indexes_on_different_tables_with_if_not_exists(cql, test_keyspace, scylla_only):
def test_two_same_name_indexes_on_different_tables_with_if_not_exists(cql, test_keyspace, scylla_only, skip_without_tablets):
schema = "p int primary key, v vector<float, 3>"
with new_test_table(cql, test_keyspace, schema) as table:
schema = "p int primary key, v vector<float, 3>"
@@ -251,8 +250,7 @@ def create_index(cql, test_keyspace, table, column):
return True
@pytest.mark.parametrize("test_keyspace", ["tablets", "vnodes"], indirect=True)
def test_try_create_cdc_with_vector_search_enabled(scylla_only, cql, test_keyspace):
def test_try_create_cdc_with_vector_search_enabled(scylla_only, cql, test_keyspace, skip_without_tablets):
schema = "pk int primary key, v vector<float, 3>"
with new_test_table(cql, test_keyspace, schema) as table:
# The vector index requires CDC to be enabled with specific options:
@@ -289,8 +287,7 @@ def test_try_create_cdc_with_vector_search_enabled(scylla_only, cql, test_keyspa
assert not alter_cdc(cql, table, {'enabled': True, 'delta': 'keys', 'postimage': False})
@pytest.mark.parametrize("test_keyspace", ["tablets", "vnodes"], indirect=True)
def test_try_disable_cdc_with_vector_search_enabled(scylla_only, cql, test_keyspace):
def test_try_disable_cdc_with_vector_search_enabled(scylla_only, cql, test_keyspace, skip_without_tablets):
schema = "pk int primary key, v vector<float, 3>"
with new_test_table(cql, test_keyspace, schema) as table:
# Enable Vector Search by creating a vector index.
@@ -301,8 +298,7 @@ def test_try_disable_cdc_with_vector_search_enabled(scylla_only, cql, test_keysp
cql.execute(f"ALTER TABLE {table} WITH cdc = {{'enabled': False}}")
@pytest.mark.parametrize("test_keyspace", ["tablets", "vnodes"], indirect=True)
def test_try_enable_vector_search_with_cdc_enabled(scylla_only, cql, test_keyspace):
def test_try_enable_vector_search_with_cdc_enabled(scylla_only, cql, test_keyspace, skip_without_tablets):
schema = "pk int primary key, v vector<float, 3>"
with new_test_table(cql, test_keyspace, schema) as table:
# The vector index requires CDC to be enabled with specific options:
@@ -330,8 +326,7 @@ def test_try_enable_vector_search_with_cdc_enabled(scylla_only, cql, test_keyspa
assert create_index(cql, test_keyspace, table, "v")
@pytest.mark.parametrize("test_keyspace", ["tablets", "vnodes"], indirect=True)
def test_try_enable_vector_search_with_cdc_disabled(scylla_only, cql, test_keyspace):
def test_try_enable_vector_search_with_cdc_disabled(scylla_only, cql, test_keyspace, skip_without_tablets):
schema = "pk int primary key, v vector<float, 3>"
with new_test_table(cql, test_keyspace, schema) as table:
# Disallow creating the vector index when CDC is explicitly disabled.
@@ -348,7 +343,7 @@ def test_try_enable_vector_search_with_cdc_disabled(scylla_only, cql, test_keysp
# It performs a vector search with tracing enabled. An exception is expected
# because the vector store node is not configured. However, due to the bug,
# Scylla crashes instead of returning an error.
def test_vector_search_when_tracing_is_enabled(cql, test_keyspace, scylla_only):
def test_vector_search_when_tracing_is_enabled(cql, test_keyspace, scylla_only, skip_without_tablets):
schema = "p int primary key, v vector<float, 3>"
with new_test_table(cql, test_keyspace, schema) as table:
cql.execute(f"CREATE CUSTOM INDEX ON {table}(v) USING 'vector_index'")

50
test/vector_search/vector_store_client_test.cc
View File

@@ -123,6 +123,12 @@ public:
}
};
cql_test_config make_config() {
cql_test_config cfg;
cfg.initial_tablets = 1;
return cfg;
}
} // namespace
BOOST_AUTO_TEST_CASE(vector_store_client_test_ctor) {
@@ -297,7 +303,7 @@ SEASTAR_TEST_CASE(vector_store_client_ann_test_disabled) {
}
SEASTAR_TEST_CASE(vector_store_client_test_ann_addr_unavailable) {
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set("http://bad.authority.here:6080");
co_await do_with_cql_env(
[](cql_test_env& env) -> future<> {
@@ -316,7 +322,7 @@ SEASTAR_TEST_CASE(vector_store_client_test_ann_addr_unavailable) {
}
SEASTAR_TEST_CASE(vector_store_client_test_ann_service_unavailable) {
auto cfg = cql_test_config();
auto cfg = make_config();
auto server = co_await make_unavailable_server();
cfg.db_config->vector_store_primary_uri.set(format("http://good.authority.here:{}", server->port()));
co_await do_with_cql_env(
@@ -339,7 +345,7 @@ SEASTAR_TEST_CASE(vector_store_client_test_ann_service_unavailable) {
}
SEASTAR_TEST_CASE(vector_store_client_test_ann_service_aborted) {
auto cfg = cql_test_config();
auto cfg = make_config();
auto server = co_await make_unavailable_server();
cfg.db_config->vector_store_primary_uri.set(format("http://good.authority.here:{}", server->port()));
co_await do_with_cql_env(
@@ -368,7 +374,7 @@ SEASTAR_TEST_CASE(vector_store_client_test_ann_service_aborted) {
SEASTAR_TEST_CASE(vector_store_client_test_ann_request) {
auto server = co_await make_vs_mock_server();
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://good.authority.here:{}", server->port()));
co_await do_with_cql_env(
[&server](cql_test_env& env) -> future<> {
@@ -516,7 +522,7 @@ SEASTAR_TEST_CASE(vector_store_client_uri_update) {
auto s1 = co_await make_vs_mock_server();
auto s2 = co_await make_vs_mock_server();
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://good.authority.here:{}", s1->port()));
co_await do_with_cql_env(
[&](cql_test_env& env) -> future<> {
@@ -550,7 +556,7 @@ SEASTAR_TEST_CASE(vector_store_client_multiple_ips_high_availability) {
auto responding_s = co_await make_vs_mock_server();
auto unavail_s = co_await make_unavailable_server(responding_s->port());
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://good.authority.here:{}", responding_s->port()));
co_await do_with_cql_env(
[&](cql_test_env& env) -> future<> {
@@ -584,7 +590,7 @@ SEASTAR_TEST_CASE(vector_store_client_multiple_ips_load_balancing) {
auto s1 = co_await make_vs_mock_server();
auto s2 = co_await make_vs_mock_server(s1->port());
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://good.authority.here:{}", s1->port()));
co_await do_with_cql_env(
[&](cql_test_env& env) -> future<> {
@@ -614,7 +620,7 @@ SEASTAR_TEST_CASE(vector_store_client_multiple_uris_high_availability) {
auto responding_s = co_await make_vs_mock_server();
auto unavail_s = co_await make_unavailable_server();
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://s1.node:{},http://s2.node:{}", unavail_s->port(), responding_s->port()));
co_await do_with_cql_env(
[&](cql_test_env& env) -> future<> {
@@ -648,7 +654,7 @@ SEASTAR_TEST_CASE(vector_store_client_multiple_uris_load_balancing) {
auto s1 = co_await make_vs_mock_server();
auto s2 = co_await make_vs_mock_server();
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://s1.node:{},http://s2.node:{}", s1->port(), s2->port()));
co_await do_with_cql_env(
[&](cql_test_env& env) -> future<> {
@@ -675,7 +681,7 @@ SEASTAR_TEST_CASE(vector_store_client_multiple_uris_load_balancing) {
SEASTAR_TEST_CASE(vector_search_metrics_test) {
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set("http://good.authority.here:6080");
co_await do_with_cql_env(
[](cql_test_env& env) -> future<> {
@@ -698,7 +704,7 @@ SEASTAR_TEST_CASE(vector_search_metrics_test) {
SEASTAR_TEST_CASE(vector_store_client_test_paging_warning) {
auto s1 = co_await make_vs_mock_server();
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://s1.node:{}", s1->port()));
co_await do_with_cql_env(
[&s1](cql_test_env& env) -> future<> {
@@ -724,7 +730,7 @@ SEASTAR_TEST_CASE(vector_store_client_test_paging_warning) {
SEASTAR_TEST_CASE(vector_store_client_test_paging_warning_doesnt_show_when_paging_disabled) {
auto s1 = co_await make_vs_mock_server();
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://s1.node:{}", s1->port()));
co_await do_with_cql_env(
[&s1](cql_test_env& env) -> future<> {
@@ -749,7 +755,7 @@ SEASTAR_TEST_CASE(vector_store_client_test_paging_warning_doesnt_show_when_pagin
SEASTAR_TEST_CASE(vector_store_client_test_paging_warning_doesnt_show_when_limit_less_than_page_size) {
auto s1 = co_await make_vs_mock_server();
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://s1.node:{}", s1->port()));
co_await do_with_cql_env(
[&s1](cql_test_env& env) -> future<> {
@@ -776,7 +782,7 @@ SEASTAR_TEST_CASE(vector_store_client_node_recovery_after_backoff) {
std::unique_ptr<vs_mock_server> avail_server;
constexpr auto HOSTNAME = "server.node";
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://{}:{}", HOSTNAME, unavail_server->port()));
co_await do_with_cql_env(
[&](cql_test_env& env) -> future<> {
@@ -815,7 +821,7 @@ SEASTAR_TEST_CASE(vector_store_client_single_status_check_after_concurrent_failu
using keys = std::expected<vector_store_client::primary_keys, vector_store_client::ann_error>;
auto unavail_s = co_await make_unavailable_server();
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://unavail.node:{}", unavail_s->port()));
co_await do_with_cql_env(
[&](cql_test_env& env) -> future<> {
@@ -858,7 +864,7 @@ SEASTAR_TEST_CASE(vector_store_client_single_status_check_after_concurrent_failu
SEASTAR_TEST_CASE(vector_store_client_updates_backoff_max_time_from_read_request_timeout_cfg) {
auto unavail_s = co_await make_unavailable_server();
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://unavail.node:{}", unavail_s->port()));
co_await do_with_cql_env(
[&](cql_test_env& env) -> future<> {
@@ -902,7 +908,7 @@ SEASTAR_TEST_CASE(vector_store_client_updates_backoff_max_time_from_read_request
SEASTAR_TEST_CASE(vector_store_client_secondary_uri) {
auto primary = co_await make_unavailable_server();
auto secondary = co_await make_vs_mock_server();
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("http://primary.node:{}", primary->port()));
cfg.db_config->vector_store_secondary_uri.set(format("http://secondary.node:{}", secondary->port()));
co_await do_with_cql_env(
@@ -927,7 +933,7 @@ SEASTAR_TEST_CASE(vector_store_client_secondary_uri) {
SEASTAR_TEST_CASE(vector_store_client_secondary_uri_only) {
auto secondary = co_await make_vs_mock_server();
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_secondary_uri.set(format("http://secondary.node:{}", secondary->port()));
co_await do_with_cql_env(
[&](cql_test_env& env) -> future<> {
@@ -950,7 +956,7 @@ SEASTAR_TEST_CASE(vector_store_client_secondary_uri_only) {
SEASTAR_TEST_CASE(vector_store_client_https) {
certificates certs;
auto server = co_await make_vs_mock_server(co_await make_server_credentials(certs));
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("https://{}:{}", certs.server_cert_cn(), server->port()));
cfg.db_config->vector_store_encryption_options.set({{"truststore", certs.ca_cert_file()}});
co_await do_with_cql_env(
@@ -976,7 +982,7 @@ SEASTAR_TEST_CASE(vector_store_client_https_rewrite_ca_cert) {
auto broken_cert = co_await seastar::make_tmp_file();
certificates certs;
auto server = co_await make_vs_mock_server(co_await make_server_credentials(certs));
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("https://{}:{}", certs.server_cert_cn(), server->port()));
cfg.db_config->vector_store_encryption_options.set({{"truststore", broken_cert.get_path().string()}});
co_await do_with_cql_env(
@@ -1022,7 +1028,7 @@ SEASTAR_TEST_CASE(vector_store_client_https_wrong_hostname) {
certificates certs;
auto server = co_await make_vs_mock_server(co_await make_server_credentials(certs));
const auto hostname = fmt::format("wrong.{}", certs.server_cert_cn());
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("https://{}:{}", hostname, server->port()));
cfg.db_config->vector_store_encryption_options.set({{"truststore", certs.ca_cert_file()}});
co_await do_with_cql_env(
@@ -1048,7 +1054,7 @@ SEASTAR_TEST_CASE(vector_store_client_https_different_ca_cert_verification_error
auto broken_cert = co_await seastar::make_tmp_file();
certificates certs;
auto server = co_await make_vs_mock_server(co_await make_server_credentials(certs));
auto cfg = cql_test_config();
auto cfg = make_config();
cfg.db_config->vector_store_primary_uri.set(format("https://{}:{}", certs.server_cert_cn(), server->port()));
cfg.db_config->vector_store_encryption_options.set({{"truststore", broken_cert.get_path().string()}});
co_await do_with_cql_env(