bahman/scylla
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

test: test_env: pick the highest sstable version by default

Browse Source 
If possible, test the highest sstable format version,
as it's the mostly used.

If there pre-written sstables we need to load from the
test directory from an older version, either specify their
version explicitly, or use the new test_env::reusable_sst
method that looks up the latest sstable version in the
given directory and generation.

Test: unit(release)
Signed-off-by: Benny Halevy <bhalevy@scylladb.com>
Message-Id: <20201210161822.2833510-1-bhalevy@scylladb.com>
This commit is contained in:
Benny Halevy
2020-12-10 18:18:22 +02:00
committed by Nadav Har'El
parent cb9e2ee00a
commit 1847d49971
9 changed files with 48 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
sstables/version.hh
View File

@@ -37,6 +37,10 @@ inline std::array<sstable_version_types, 4> all_sstable_versions = {
sstable_version_types::md,
};
inline auto get_highest_sstable_version() {
return all_sstable_versions[all_sstable_versions.size() - 1];
}
inline sstable_version_types from_string(const seastar::sstring& format) {
if (format == "ka") {
return sstable_version_types::ka;

6
test/boost/mutation_reader_test.cc
View File

@@ -566,7 +566,7 @@ struct sst_factory {
{}
sstables::shared_sstable operator()() {
auto sst = env.make_sstable(s, path, gen, sstables::sstable::version_types::la, sstables::sstable::format_types::big);
auto sst = env.make_sstable(s, path, gen);
sst->set_sstable_level(level);
return sst;
@@ -891,7 +891,7 @@ sstables::shared_sstable create_sstable(sstables::test_env& env, simple_schema&
}
return make_sstable_containing([&] {
return env.make_sstable(sschema.schema(), path, 0, sstables::sstable::version_types::la, sstables::sstable::format_types::big);
return env.make_sstable(sschema.schema(), path, 0);
}
, mutations);
}
@@ -901,7 +901,7 @@ sstables::shared_sstable create_sstable(sstables::test_env& env, schema_ptr s, s
static thread_local auto tmp = tmpdir();
static int gen = 0;
return make_sstable_containing([&] {
return env.make_sstable(s, tmp.path().string(), gen++, sstables::sstable::version_types::la, sstables::sstable::format_types::big);
return env.make_sstable(s, tmp.path().string(), gen++);
}, mutations);
}

4
test/boost/sstable_datafile_test.cc
View File

@@ -5294,7 +5294,7 @@ SEASTAR_TEST_CASE(sstable_scrub_test) {
auto tmp = tmpdir();
auto sst_gen = [&env, schema, &tmp, gen = make_lw_shared<unsigned>(1)] () mutable {
return env.make_sstable(schema, tmp.path().string(), (*gen)++, sstables::sstable_version_types::mc, big);
return env.make_sstable(schema, tmp.path().string(), (*gen)++);
};
std::vector<mutation_fragment> corrupt_fragments;
@@ -6734,7 +6734,7 @@ SEASTAR_TEST_CASE(stcs_reshape_test) {
std::vector<shared_sstable> sstables;
sstables.reserve(s->max_compaction_threshold());
for (auto gen = 1; gen <= s->max_compaction_threshold(); gen++) {
auto sst = env.make_sstable(s, "", gen, la, big);
auto sst = env.make_sstable(s, "", gen);
sstables::test(sst).set_data_file_size(1);
sstables.push_back(std::move(sst));
}

11
test/boost/sstable_directory_test.cc
View File

@@ -71,7 +71,7 @@ make_sstable_for_this_shard(std::function<sstables::shared_sstable()> sst_factor
/// Arguments passed to the function are passed to table::make_sstable
template <typename... Args>
sstables::shared_sstable
make_sstable_for_all_shards(database& db, table& table, fs::path sstdir, int64_t generation, Args&&... args) {
make_sstable_for_all_shards(database& db, table& table, fs::path sstdir, int64_t generation) {
// Unlike the previous helper, we'll assume we're in a thread here. It's less flexible
// but the users are usually in a thread, and rewrite_toc_without_scylla_component requires
// a thread. We could fix that, but deferring that for now.
@@ -85,7 +85,8 @@ make_sstable_for_all_shards(database& db, table& table, fs::path sstdir, int64_t
m.set_clustered_cell(clustering_key::make_empty(), bytes("c"), data_value(int32_t(0)), api::timestamp_type(0));
mt->apply(std::move(m));
}
auto sst = table.make_sstable(sstdir.native(), generation++, args...);
auto sst = table.make_sstable(sstdir.native(), generation++,
sstables::get_highest_sstable_version(), sstables::sstable::format_types::big);
write_memtable_to_sstable(*mt, sst, table.get_sstables_manager().configure_writer()).get();
mt->clear_gently().get();
// We can't write an SSTable with bad sharding, so pretend
@@ -482,7 +483,7 @@ SEASTAR_TEST_CASE(sstable_directory_shared_sstables_reshard_correctly) {
unsigned num_sstables = 10 * smp::count;
auto generation = 0;
for (unsigned nr = 0; nr < num_sstables; ++nr) {
make_sstable_for_all_shards(e.db().local(), cf, upload_path.native(), generation++, sstables::sstable_version_types::mc, sstables::sstable::format_types::big);
make_sstable_for_all_shards(e.db().local(), cf, upload_path.native(), generation++);
}
with_sstable_directory(upload_path, 1,
@@ -531,7 +532,7 @@ SEASTAR_TEST_CASE(sstable_directory_shared_sstables_reshard_distributes_well_eve
unsigned num_sstables = 10 * smp::count;
auto generation = 0;
for (unsigned nr = 0; nr < num_sstables; ++nr) {
make_sstable_for_all_shards(e.db().local(), cf, upload_path.native(), generation++ * smp::count, sstables::sstable_version_types::mc, sstables::sstable::format_types::big);
make_sstable_for_all_shards(e.db().local(), cf, upload_path.native(), generation++ * smp::count);
}
with_sstable_directory(upload_path, 1,
@@ -580,7 +581,7 @@ SEASTAR_TEST_CASE(sstable_directory_shared_sstables_reshard_respect_max_threshol
unsigned num_sstables = (cf.schema()->max_compaction_threshold() + 1) * smp::count;
auto generation = 0;
for (unsigned nr = 0; nr < num_sstables; ++nr) {
make_sstable_for_all_shards(e.db().local(), cf, upload_path.native(), generation++, sstables::sstable_version_types::mc, sstables::sstable::format_types::big);
make_sstable_for_all_shards(e.db().local(), cf, upload_path.native(), generation++);
}
with_sstable_directory(upload_path, 1,

4
test/boost/sstable_mutation_test.cc
View File

@@ -965,9 +965,7 @@ SEASTAR_TEST_CASE(test_promoted_index_blocks_are_monotonic) {
auto sst = env.make_sstable(s,
dir.path().string(),
1 /* generation */,
sstables::sstable::version_types::ka,
sstables::sstable::format_types::big);
1 /* generation */);
sstable_writer_config cfg = env.manager().configure_writer();
cfg.promoted_index_block_size = 1;
sst->write_components(mt->make_flat_reader(s, tests::make_permit()), 1, s, cfg, mt->get_encoding_stats()).get();

4
test/boost/sstable_test.cc
View File

@@ -173,7 +173,7 @@ SEASTAR_TEST_CASE(missing_summary_first_last_sane) {
}
static future<sstable_ptr> do_write_sst(test_env& env, schema_ptr schema, sstring load_dir, sstring write_dir, unsigned long generation) {
return env.reusable_sst(std::move(schema), load_dir, generation, la).then([write_dir, generation] (sstable_ptr sst) {
return env.reusable_sst(std::move(schema), load_dir, generation).then([write_dir, generation] (sstable_ptr sst) {
sstables::test(sst).change_generation_number(generation + 1);
sstables::test(sst).change_dir(write_dir);
auto fut = sstables::test(sst).store();
@@ -245,7 +245,7 @@ future<>
write_and_validate_sst(schema_ptr s, sstring dir, noncopyable_function<future<> (shared_sstable sst1, shared_sstable sst2)> func) {
return test_env::do_with(tmpdir(), [s = std::move(s), dir = std::move(dir), func = std::move(func)] (test_env& env, tmpdir& tmp) mutable {
return do_write_sst(env, s, dir, tmp.path().string(), 1).then([&env, &tmp, s = std::move(s), func = std::move(func)] (auto sst1) {
auto sst2 = env.make_sstable(s, tmp.path().string(), 2, la, big);
auto sst2 = env.make_sstable(s, tmp.path().string(), 2, sst1->get_version());
return func(std::move(sst1), std::move(sst2));
});
});

7
test/lib/sstable_test_env.hh
View File

@@ -43,19 +43,22 @@ public:
}
shared_sstable make_sstable(schema_ptr schema, sstring dir, unsigned long generation,
sstable::version_types v, sstable::format_types f = sstable::format_types::big,
sstable::version_types v = sstables::get_highest_sstable_version(), sstable::format_types f = sstable::format_types::big,
size_t buffer_size = default_sstable_buffer_size, gc_clock::time_point now = gc_clock::now()) {
return _mgr->make_sstable(std::move(schema), dir, generation, v, f, now, default_io_error_handler_gen(), buffer_size);
}
future<shared_sstable> reusable_sst(schema_ptr schema, sstring dir, unsigned long generation,
sstable::version_types version = sstable::version_types::la, sstable::format_types f = sstable::format_types::big) {
sstable::version_types version, sstable::format_types f = sstable::format_types::big) {
auto sst = make_sstable(std::move(schema), dir, generation, version, f);
return sst->load().then([sst = std::move(sst)] {
return make_ready_future<shared_sstable>(std::move(sst));
});
}
// looks up the sstable in the given dir
future<shared_sstable> reusable_sst(schema_ptr schema, sstring dir, unsigned long generation);
sstables_manager& manager() { return *_mgr; }
future<> working_sst(schema_ptr schema, sstring dir, unsigned long generation) {

22
test/lib/sstable_utils.cc
View File

@@ -164,3 +164,25 @@ future<compaction_info> compact_sstables(sstables::compaction_descriptor descrip
std::vector<std::pair<sstring, dht::token>> token_generation_for_current_shard(unsigned tokens_to_generate) {
return token_generation_for_shard(tokens_to_generate, this_shard_id());
}
future<shared_sstable> test_env::reusable_sst(schema_ptr schema, sstring dir, unsigned long generation) {
return seastar::do_with(shared_sstable(), std::exception_ptr(), std::move(dir), all_sstable_versions.rbegin(),
[this, schema, generation] (shared_sstable& ret_sst, std::exception_ptr& ret_ep, sstring& dir, auto& v) {
return do_until([&] { return ret_sst || v == all_sstable_versions.rend(); }, [this, schema, generation, &ret_sst, &ret_ep, &dir, &v] {
return reusable_sst(schema, dir, generation, *v++).then_wrapped([&] (future<shared_sstable> f) {
if (f.failed()) {
ret_ep = f.get_exception();
} else {
ret_sst = f.get0();
}
});
}).then([&] {
if (ret_sst) {
return make_ready_future<shared_sstable>(std::move(ret_sst));
} else {
assert(ret_ep);
return make_exception_future<shared_sstable>(std::move(ret_ep));
}
});
});
}

6
test/perf/perf_sstable.hh
View File

@@ -122,7 +122,7 @@ public:
}
future<> load_sstables(unsigned iterations) {
_sst.push_back(_env.make_sstable(s, this->dir(), 0, sstable::version_types::ka, sstable::format_types::big));
_sst.push_back(_env.make_sstable(s, this->dir(), 0, sstables::get_highest_sstable_version(), sstable::format_types::big));
return _sst.back()->load();
}
@@ -138,7 +138,7 @@ public:
size_t partitions = _mt->partition_count();
test_setup::create_empty_test_dir(dir()).get();
auto sst = _env.make_sstable(s, dir(), idx, sstable::version_types::ka, sstable::format_types::big, _cfg.buffer_size);
auto sst = _env.make_sstable(s, dir(), idx, sstables::get_highest_sstable_version(), sstable::format_types::big, _cfg.buffer_size);
auto start = perf_sstable_test_env::now();
write_memtable_to_sstable_for_test(*_mt, sst).get();
@@ -155,7 +155,7 @@ public:
return test_setup::create_empty_test_dir(dir()).then([this, idx] {
return sstables::test_env::do_with_async_returning<double>([this, idx] (sstables::test_env& env) {
auto sst_gen = [this, gen = make_lw_shared<unsigned>(idx)] () mutable {
return _env.make_sstable(s, dir(), (*gen)++, sstable::version_types::ka, sstable::format_types::big, _cfg.buffer_size);
return _env.make_sstable(s, dir(), (*gen)++, sstables::get_highest_sstable_version(), sstable::format_types::big, _cfg.buffer_size);
};
std::vector<shared_sstable> ssts;