diff --git a/compaction/compaction_strategy.cc b/compaction/compaction_strategy.cc index 9be7910184..a86db5cd24 100644 --- a/compaction/compaction_strategy.cc +++ b/compaction/compaction_strategy.cc @@ -24,7 +24,6 @@ #include #include #include "size_tiered_compaction_strategy.hh" -#include "date_tiered_compaction_strategy.hh" #include "leveled_compaction_strategy.hh" #include "time_window_compaction_strategy.hh" #include "backlog_controller.hh" @@ -32,7 +31,6 @@ #include "size_tiered_backlog_tracker.hh" #include "leveled_manifest.hh" -logging::logger date_tiered_manifest::logger = logging::logger("DateTieredCompactionStrategy"); logging::logger leveled_manifest::logger("LeveledManifest"); namespace sstables { @@ -499,195 +497,8 @@ std::unique_ptr time_window_compaction_strateg } // namespace sstables -std::vector -date_tiered_manifest::get_next_sstables(table_state& table_s, std::vector& uncompacting, gc_clock::time_point compaction_time) { - if (table_s.main_sstable_set().all()->empty()) { - return {}; - } - - // Find fully expired SSTables. Those will be included no matter what. - auto expired = table_s.fully_expired_sstables(uncompacting, compaction_time); - - if (!expired.empty()) { - auto is_expired = [&] (const sstables::shared_sstable& s) { return expired.contains(s); }; - uncompacting.erase(boost::remove_if(uncompacting, is_expired), uncompacting.end()); - } - - auto compaction_candidates = get_next_non_expired_sstables(table_s, uncompacting, compaction_time); - if (!expired.empty()) { - compaction_candidates.insert(compaction_candidates.end(), expired.begin(), expired.end()); - } - return compaction_candidates; -} - -int64_t date_tiered_manifest::get_estimated_tasks(table_state& table_s) const { - int base = table_s.schema()->min_compaction_threshold(); - int64_t now = get_now(table_s.main_sstable_set().all()); - std::vector sstables; - int64_t n = 0; - - auto all_sstables = table_s.main_sstable_set().all(); - sstables.reserve(all_sstables->size()); - for (auto& entry : *all_sstables) { - sstables.push_back(entry); - } - auto candidates = filter_old_sstables(sstables, _options.max_sstable_age, now); - auto buckets = get_buckets(create_sst_and_min_timestamp_pairs(candidates), _options.base_time, base, now); - - for (auto& bucket : buckets) { - if (bucket.size() >= size_t(table_s.schema()->min_compaction_threshold())) { - n += std::ceil(double(bucket.size()) / table_s.schema()->max_compaction_threshold()); - } - } - return n; -} - -std::vector -date_tiered_manifest::get_next_non_expired_sstables(table_state& table_s, std::vector& non_expiring_sstables, gc_clock::time_point compaction_time) { - int base = table_s.schema()->min_compaction_threshold(); - int64_t now = get_now(table_s.main_sstable_set().all()); - auto most_interesting = get_compaction_candidates(table_s, non_expiring_sstables, now, base); - - return most_interesting; - - // FIXME: implement functionality below that will look for a single sstable with worth dropping tombstone, - // iff strategy didn't find anything to compact. So it's not essential. -#if 0 - // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone - // ratio is greater than threshold. - - List sstablesWithTombstones = Lists.newArrayList(); - for (SSTableReader sstable : nonExpiringSSTables) - { - if (worthDroppingTombstones(sstable, gcBefore)) - sstablesWithTombstones.add(sstable); - } - if (sstablesWithTombstones.isEmpty()) - return Collections.emptyList(); - - return Collections.singletonList(Collections.min(sstablesWithTombstones, new SSTableReader.SizeComparator())); -#endif -} - -std::vector -date_tiered_manifest::get_compaction_candidates(table_state& table_s, std::vector candidate_sstables, int64_t now, int base) { - int min_threshold = table_s.schema()->min_compaction_threshold(); - int max_threshold = table_s.schema()->max_compaction_threshold(); - auto candidates = filter_old_sstables(candidate_sstables, _options.max_sstable_age, now); - - auto buckets = get_buckets(create_sst_and_min_timestamp_pairs(candidates), _options.base_time, base, now); - - return newest_bucket(buckets, min_threshold, max_threshold, now, _options.base_time); -} - -int64_t date_tiered_manifest::get_now(lw_shared_ptr shared_set) { - int64_t max_timestamp = 0; - for (auto& sst : *shared_set) { - int64_t candidate = sst->get_stats_metadata().max_timestamp; - max_timestamp = candidate > max_timestamp ? candidate : max_timestamp; - } - return max_timestamp; -} - -std::vector -date_tiered_manifest::filter_old_sstables(std::vector sstables, api::timestamp_type max_sstable_age, int64_t now) { - if (max_sstable_age == 0) { - return sstables; - } - int64_t cutoff = now - max_sstable_age; - - std::erase_if(sstables, [cutoff] (auto& sst) { - return sst->get_stats_metadata().max_timestamp < cutoff; - }); - - return sstables; -} - -std::vector> -date_tiered_manifest::create_sst_and_min_timestamp_pairs(const std::vector& sstables) { - std::vector> sstable_min_timestamp_pairs; - sstable_min_timestamp_pairs.reserve(sstables.size()); - for (auto& sst : sstables) { - sstable_min_timestamp_pairs.emplace_back(sst, sst->get_stats_metadata().min_timestamp); - } - return sstable_min_timestamp_pairs; -} - -date_tiered_compaction_strategy_options::date_tiered_compaction_strategy_options(const std::map& options) { - using namespace cql3::statements; - - auto tmp_value = sstables::compaction_strategy_impl::get_value(options, TIMESTAMP_RESOLUTION_KEY); - auto target_unit = tmp_value ? tmp_value.value() : DEFAULT_TIMESTAMP_RESOLUTION; - - tmp_value = sstables::compaction_strategy_impl::get_value(options, MAX_SSTABLE_AGE_KEY); - auto fractional_days = property_definitions::to_double(MAX_SSTABLE_AGE_KEY, tmp_value, DEFAULT_MAX_SSTABLE_AGE_DAYS); - int64_t max_sstable_age_in_hours = std::lround(fractional_days * 24); - max_sstable_age = duration_conversor::convert(target_unit, std::chrono::hours(max_sstable_age_in_hours)); - - tmp_value = sstables::compaction_strategy_impl::get_value(options, BASE_TIME_KEY); - auto base_time_seconds = property_definitions::to_long(BASE_TIME_KEY, tmp_value, DEFAULT_BASE_TIME_SECONDS); - base_time = duration_conversor::convert(target_unit, std::chrono::seconds(base_time_seconds)); -} - -date_tiered_compaction_strategy_options::date_tiered_compaction_strategy_options() { - auto max_sstable_age_in_hours = int64_t(DEFAULT_MAX_SSTABLE_AGE_DAYS * 24); - max_sstable_age = std::chrono::duration_cast(std::chrono::hours(max_sstable_age_in_hours)).count(); - base_time = std::chrono::duration_cast(std::chrono::seconds(DEFAULT_BASE_TIME_SECONDS)).count(); -} - namespace sstables { -date_tiered_compaction_strategy::date_tiered_compaction_strategy(const std::map& options) - : compaction_strategy_impl(options) - , _manifest(options) -{ - clogger.warn("DateTieredCompactionStrategy is deprecated. Usually cases for which it is used are better handled by TimeWindowCompactionStrategy." - " Please change your compaction strategy to TWCS as DTCS will be retired in the near future"); - - // tombstone compaction is disabled by default because: - // - deletion shouldn't be used with DTCS; rather data is deleted through TTL. - // - with time series workloads, it's usually better to wait for whole sstable to be expired rather than - // compacting a single sstable when it's more than 20% (default value) expired. - // For more details, see CASSANDRA-9234 - if (!options.contains(TOMBSTONE_COMPACTION_INTERVAL_OPTION) && !options.contains(TOMBSTONE_THRESHOLD_OPTION)) { - _disable_tombstone_compaction = true; - date_tiered_manifest::logger.debug("Disabling tombstone compactions for DTCS"); - } else { - date_tiered_manifest::logger.debug("Enabling tombstone compactions for DTCS"); - } - - _use_clustering_key_filter = true; -} - -compaction_descriptor date_tiered_compaction_strategy::get_sstables_for_compaction(table_state& table_s, strategy_control& control, std::vector candidates) { - auto compaction_time = gc_clock::now(); - auto sstables = _manifest.get_next_sstables(table_s, candidates, compaction_time); - - if (!sstables.empty()) { - date_tiered_manifest::logger.debug("datetiered: Compacting {} out of {} sstables", sstables.size(), candidates.size()); - return sstables::compaction_descriptor(std::move(sstables)); - } - - // filter out sstables which droppable tombstone ratio isn't greater than the defined threshold. - auto e = boost::range::remove_if(candidates, [this, compaction_time, &table_s] (const sstables::shared_sstable& sst) -> bool { - return !worth_dropping_tombstones(sst, compaction_time, table_s.get_tombstone_gc_state()); - }); - candidates.erase(e, candidates.end()); - if (candidates.empty()) { - return sstables::compaction_descriptor(); - } - // find oldest sstable which is worth dropping tombstones because they are more unlikely to - // shadow data from other sstables, and it also tends to be relatively big. - auto it = std::min_element(candidates.begin(), candidates.end(), [] (auto& i, auto& j) { - return i->get_stats_metadata().min_timestamp < j->get_stats_metadata().min_timestamp; - }); - return sstables::compaction_descriptor({ *it }); -} - -std::unique_ptr date_tiered_compaction_strategy::make_backlog_tracker() const { - return std::make_unique(); -} - size_tiered_compaction_strategy::size_tiered_compaction_strategy(const std::map& options) : compaction_strategy_impl(options) , _options(options) @@ -775,9 +586,6 @@ compaction_strategy make_compaction_strategy(compaction_strategy_type strategy, case compaction_strategy_type::leveled: impl = ::make_shared(options); break; - case compaction_strategy_type::date_tiered: - impl = ::make_shared(options); - break; case compaction_strategy_type::time_window: impl = ::make_shared(options); break; @@ -796,7 +604,6 @@ compaction_strategy_state compaction_strategy_state::make(const compaction_strat switch (cs.type()) { case compaction_strategy_type::null: case compaction_strategy_type::size_tiered: - case compaction_strategy_type::date_tiered: return compaction_strategy_state(default_empty_state{}); case compaction_strategy_type::leveled: return compaction_strategy_state(leveled_compaction_strategy_state{}); diff --git a/compaction/compaction_strategy.hh b/compaction/compaction_strategy.hh index b3e37d5c42..212f59b5a8 100644 --- a/compaction/compaction_strategy.hh +++ b/compaction/compaction_strategy.hh @@ -71,8 +71,6 @@ public: return "SizeTieredCompactionStrategy"; case compaction_strategy_type::leveled: return "LeveledCompactionStrategy"; - case compaction_strategy_type::date_tiered: - return "DateTieredCompactionStrategy"; case compaction_strategy_type::time_window: return "TimeWindowCompactionStrategy"; default: @@ -89,8 +87,6 @@ public: return compaction_strategy_type::size_tiered; } else if (short_name == "LeveledCompactionStrategy") { return compaction_strategy_type::leveled; - } else if (short_name == "DateTieredCompactionStrategy") { - return compaction_strategy_type::date_tiered; } else if (short_name == "TimeWindowCompactionStrategy") { return compaction_strategy_type::time_window; } else { diff --git a/compaction/compaction_strategy_type.hh b/compaction/compaction_strategy_type.hh index 73fa4941c6..6da2e75953 100644 --- a/compaction/compaction_strategy_type.hh +++ b/compaction/compaction_strategy_type.hh @@ -14,7 +14,6 @@ enum class compaction_strategy_type { null, size_tiered, leveled, - date_tiered, time_window, }; diff --git a/compaction/date_tiered_compaction_strategy.hh b/compaction/date_tiered_compaction_strategy.hh deleted file mode 100644 index 7dad7b0ba4..0000000000 --- a/compaction/date_tiered_compaction_strategy.hh +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Copyright (C) 2016-present-2017 ScyllaDB - * - * Modified by ScyllaDB - */ - -/* - * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0) - */ - -#pragma once - -#include -#include -#include -#include -#include -#include "sstables/sstables.hh" -#include "compaction.hh" -#include "timestamp.hh" -#include "cql3/statements/property_definitions.hh" -#include "compaction_strategy_impl.hh" - -static constexpr double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365; -static constexpr int64_t DEFAULT_BASE_TIME_SECONDS = 60; - -struct duration_conversor { - // Convert given duration to TargetDuration and return value as timestamp. - template - static api::timestamp_type convert(SourceDuration d) { - return std::chrono::duration_cast(d).count(); - } - - // Convert given duration to duration that is represented by the string - // target_duration, and return value as timestamp. - template - static api::timestamp_type convert(const sstring& target_duration, SourceDuration d) { - if (target_duration == "HOURS") { - return convert(d); - } else if (target_duration == "MICROSECONDS") { - return convert(d); - } else if (target_duration == "MILLISECONDS") { - return convert(d); - } else if (target_duration == "MINUTES") { - return convert(d); - } else if (target_duration == "NANOSECONDS") { - return convert(d); - } else if (target_duration == "SECONDS") { - return convert(d); - } else { - throw std::runtime_error(format("target duration {} is not available", target_duration)); - } - } -}; - -class date_tiered_compaction_strategy_options { - const sstring DEFAULT_TIMESTAMP_RESOLUTION = "MICROSECONDS"; - const sstring TIMESTAMP_RESOLUTION_KEY = "timestamp_resolution"; - const sstring MAX_SSTABLE_AGE_KEY = "max_sstable_age_days"; - const sstring BASE_TIME_KEY = "base_time_seconds"; - - api::timestamp_type max_sstable_age; - api::timestamp_type base_time; -public: - date_tiered_compaction_strategy_options(const std::map& options); - - date_tiered_compaction_strategy_options(); -private: - - friend class date_tiered_manifest; -}; - -class date_tiered_manifest { - date_tiered_compaction_strategy_options _options; -public: - static logging::logger logger; - - date_tiered_manifest() = delete; - - date_tiered_manifest(const std::map& options) - : _options(options) {} - - std::vector - get_next_sstables(table_state& table_s, std::vector& uncompacting, gc_clock::time_point compaction_time); - - int64_t get_estimated_tasks(table_state& table_s) const; -private: - std::vector - get_next_non_expired_sstables(table_state& table_s, std::vector& non_expiring_sstables, gc_clock::time_point compaction_time); - - std::vector - get_compaction_candidates(table_state& table_s, std::vector candidate_sstables, int64_t now, int base); - - /** - * Gets the timestamp that DateTieredCompactionStrategy considers to be the "current time". - * @return the maximum timestamp across all SSTables. - */ - static int64_t get_now(lw_shared_ptr shared_set); - - /** - * Removes all sstables with max timestamp older than maxSSTableAge. - * @return a list of sstables with the oldest sstables excluded - */ - static std::vector - filter_old_sstables(std::vector sstables, api::timestamp_type max_sstable_age, int64_t now); - - /** - * - * @param sstables - * @return - */ - static std::vector> - create_sst_and_min_timestamp_pairs(const std::vector& sstables); - - /** - * A target time span used for bucketing SSTables based on timestamps. - */ - struct target { - // How big a range of timestamps fit inside the target. - int64_t size; - // A timestamp t hits the target iff t / size == divPosition. - int64_t div_position; - - target() = delete; - target(int64_t size, int64_t div_position) : size(size), div_position(div_position) {} - - /** - * Compares the target to a timestamp. - * @param timestamp the timestamp to compare. - * @return a negative integer, zero, or a positive integer as the target lies before, covering, or after than the timestamp. - */ - int compare_to_timestamp(int64_t timestamp) { - auto ts1 = div_position; - auto ts2 = timestamp / size; - return (ts1 > ts2 ? 1 : (ts1 == ts2 ? 0 : -1)); - } - - /** - * Tells if the timestamp hits the target. - * @param timestamp the timestamp to test. - * @return true iff timestamp / size == divPosition. - */ - bool on_target(int64_t timestamp) { - return compare_to_timestamp(timestamp) == 0; - } - - /** - * Gets the next target, which represents an earlier time span. - * @param base The number of contiguous targets that will have the same size. Targets following those will be base times as big. - * @return - */ - target next_target(int base) - { - if (div_position % base > 0) { - return target(size, div_position - 1); - } else { - return target(size * base, div_position / base - 1); - } - } - }; - - - /** - * Group files with similar min timestamp into buckets. Files with recent min timestamps are grouped together into - * buckets designated to short timespans while files with older timestamps are grouped into buckets representing - * longer timespans. - * @param files pairs consisting of a file and its min timestamp - * @param timeUnit - * @param base - * @param now - * @return a list of buckets of files. The list is ordered such that the files with newest timestamps come first. - * Each bucket is also a list of files ordered from newest to oldest. - */ - std::vector> - get_buckets(std::vector>&& files, api::timestamp_type time_unit, int base, int64_t now) const { - // Sort files by age. Newest first. - std::sort(files.begin(), files.end(), [] (auto& i, auto& j) { - return i.second > j.second; - }); - - std::vector> buckets; - auto target = get_initial_target(now, time_unit); - auto it = files.begin(); - - while (it != files.end()) { - bool finish = false; - while (!target.on_target(it->second)) { - // If the file is too new for the target, skip it. - if (target.compare_to_timestamp(it->second) < 0) { - it++; - if (it == files.end()) { - finish = true; - break; - } - } else { // If the file is too old for the target, switch targets. - target = target.next_target(base); - } - } - if (finish) { - break; - } - - std::vector bucket; - while (target.on_target(it->second)) { - bucket.push_back(it->first); - it++; - if (it == files.end()) { - break; - } - } - buckets.push_back(bucket); - } - - return buckets; - } - - target get_initial_target(uint64_t now, int64_t time_unit) const { - return target(time_unit, now / time_unit); - } - - /** - * @param buckets list of buckets, sorted from newest to oldest, from which to return the newest bucket within thresholds. - * @param minThreshold minimum number of sstables in a bucket to qualify. - * @param maxThreshold maximum number of sstables to compact at once (the returned bucket will be trimmed down to this). - * @return a bucket (list) of sstables to compact. - */ - std::vector - newest_bucket(std::vector>& buckets, int min_threshold, int max_threshold, - int64_t now, api::timestamp_type base_time) { - - // If the "incoming window" has at least minThreshold SSTables, choose that one. - // For any other bucket, at least 2 SSTables is enough. - // In any case, limit to maxThreshold SSTables. - target incoming_window = get_initial_target(now, base_time); - for (auto& bucket : buckets) { - auto min_timestamp = bucket.front()->get_stats_metadata().min_timestamp; - if (bucket.size() >= size_t(min_threshold) || - (bucket.size() >= 2 && !incoming_window.on_target(min_timestamp))) { - trim_to_threshold(bucket, max_threshold); - return bucket; - } - } - return {}; - } - - - /** - * @param bucket list of sstables, ordered from newest to oldest by getMinTimestamp(). - * @param maxThreshold maximum number of sstables in a single compaction task. - * @return A bucket trimmed to the maxThreshold newest sstables. - */ - static void trim_to_threshold(std::vector& bucket, int max_threshold) { - // Trim the oldest sstables off the end to meet the maxThreshold - bucket.resize(std::min(bucket.size(), size_t(max_threshold))); - } -}; - -namespace sstables { - -class date_tiered_compaction_strategy : public compaction_strategy_impl { - date_tiered_manifest _manifest; -public: - date_tiered_compaction_strategy(const std::map& options); - virtual compaction_descriptor get_sstables_for_compaction(table_state& table_s, strategy_control& control, std::vector candidates) override; - - virtual int64_t estimated_pending_compactions(table_state& table_s) const override { - return _manifest.get_estimated_tasks(table_s); - } - - virtual compaction_strategy_type type() const override { - return compaction_strategy_type::date_tiered; - } - - virtual std::unique_ptr make_backlog_tracker() const override; -}; - -} diff --git a/cql3/statements/create_table_statement.cc b/cql3/statements/create_table_statement.cc index ea409ab2f6..e1f9e5b654 100644 --- a/cql3/statements/create_table_statement.cc +++ b/cql3/statements/create_table_statement.cc @@ -445,24 +445,6 @@ std::optional check_restricted_table_properties( // Evaluate whether the strategy to evaluate was explicitly passed auto cs = (strategy) ? strategy : current_strategy; - if (strategy && *strategy == sstables::compaction_strategy_type::date_tiered) { - switch(qp.db().get_config().restrict_dtcs()) { - case db::tri_mode_restriction_t::mode::TRUE: - throw exceptions::configuration_exception( - "DateTieredCompactionStrategy is deprecated, and " - "forbidden by the current configuration. Please use " - "TimeWindowCompactionStrategy instead. You may also override this " - "restriction by setting the restrict_dtcs configuration option " - "to false."); - case db::tri_mode_restriction_t::mode::WARN: - return format("DateTieredCompactionStrategy is deprecated, " - "but was used for table {}.{}. The restrict_dtcs " - "configuration option can be changed to silence this warning " - " or make it into an error.", keyspace, table); - case db::tri_mode_restriction_t::mode::FALSE: - break; - } - } if (cs == sstables::compaction_strategy_type::time_window) { std::map options = (strategy) ? cfprops.get_compaction_type_options() : (*schema)->compaction_strategy_options(); sstables::time_window_compaction_strategy_options twcs_options(options); diff --git a/db/config.cc b/db/config.cc index 9783722d27..0b47655d56 100644 --- a/db/config.cc +++ b/db/config.cc @@ -983,7 +983,7 @@ db::config::config(std::shared_ptr exts) , flush_schema_tables_after_modification(this, "flush_schema_tables_after_modification", liveness::LiveUpdate, value_status::Used, true, "Flush tables in the system_schema keyspace after schema modification. This is required for crash recovery, but slows down tests and can be disabled for them") , restrict_replication_simplestrategy(this, "restrict_replication_simplestrategy", liveness::LiveUpdate, value_status::Used, db::tri_mode_restriction_t::mode::FALSE, "Controls whether to disable SimpleStrategy replication. Can be true, false, or warn.") - , restrict_dtcs(this, "restrict_dtcs", liveness::LiveUpdate, value_status::Used, db::tri_mode_restriction_t::mode::TRUE, "Controls whether to prevent setting DateTieredCompactionStrategy. Can be true, false, or warn.") + , restrict_dtcs(this, "restrict_dtcs", liveness::LiveUpdate, value_status::Unused, db::tri_mode_restriction_t::mode::TRUE, "Controls whether to prevent setting DateTieredCompactionStrategy. Can be true, false, or warn.") , restrict_twcs_without_default_ttl(this, "restrict_twcs_without_default_ttl", liveness::LiveUpdate, value_status::Used, db::tri_mode_restriction_t::mode::WARN, "Controls whether to prevent creating TimeWindowCompactionStrategy tables without a default TTL. Can be true, false, or warn.") , restrict_future_timestamp(this, "restrict_future_timestamp",liveness::LiveUpdate, value_status::Used, true, "Controls whether to detect and forbid unreasonable USING TIMESTAMP, more than 3 days into the future.") , ignore_truncation_record(this, "unsafe_ignore_truncation_record", value_status::Used, false, diff --git a/test/boost/sstable_compaction_test.cc b/test/boost/sstable_compaction_test.cc index 33dc8fa40a..224a5a109c 100644 --- a/test/boost/sstable_compaction_test.cc +++ b/test/boost/sstable_compaction_test.cc @@ -37,7 +37,6 @@ #include "range.hh" #include "partition_slice_builder.hh" #include "compaction/compaction_strategy_impl.hh" -#include "compaction/date_tiered_compaction_strategy.hh" #include "compaction/time_window_compaction_strategy.hh" #include "compaction/leveled_compaction_strategy.hh" #include "test/lib/mutation_assertions.hh" @@ -1312,102 +1311,6 @@ SEASTAR_TEST_CASE(compaction_with_fully_expired_table) { }); } -SEASTAR_TEST_CASE(basic_date_tiered_strategy_test) { - return test_env::do_with_async([] (test_env& env) { - schema_builder builder(make_shared_schema({}, some_keyspace, some_column_family, - {{"p1", utf8_type}}, {}, {}, {}, utf8_type)); - builder.set_min_compaction_threshold(4); - auto s = builder.build(schema_builder::compact_storage::no); - auto cf = env.make_table_for_tests(s); - auto stop_cf = deferred_stop(cf); - - std::vector candidates; - std::unordered_set expected; - int min_threshold = cf->schema()->min_compaction_threshold(); - auto now = db_clock::now(); - auto past_hour = now - std::chrono::seconds(3600); - int64_t timestamp_for_now = now.time_since_epoch().count() * 1000; - int64_t timestamp_for_past_hour = past_hour.time_since_epoch().count() * 1000; - - const auto key = tests::generate_partition_key(s); - for (auto i = 1; i <= min_threshold; i++) { - auto sst = add_sstable_for_overlapping_test(env, cf, key.key(), key.key(), - build_stats(timestamp_for_now, timestamp_for_now, std::numeric_limits::max())); - candidates.push_back(sst); - expected.insert(sst); - } - // add sstable that belong to a different time tier. - auto sst = add_sstable_for_overlapping_test(env, cf, key.key(), key.key(), - build_stats(timestamp_for_past_hour, timestamp_for_past_hour, std::numeric_limits::max())); - candidates.push_back(sst); - - auto gc_before = gc_clock::now() - cf->schema()->gc_grace_seconds(); - std::map options; - date_tiered_manifest manifest(options); - auto sstables = manifest.get_next_sstables(cf.as_table_state(), candidates, gc_before); - BOOST_REQUIRE(sstables.size() == 4); - for (auto& sst : sstables) { - BOOST_REQUIRE(expected.erase(sst)); - } - BOOST_REQUIRE(expected.empty()); - }); -} - -SEASTAR_TEST_CASE(date_tiered_strategy_test_2) { - return test_env::do_with_async([] (test_env& env) { - schema_builder builder(make_shared_schema({}, some_keyspace, some_column_family, - {{"p1", utf8_type}}, {}, {}, {}, utf8_type)); - builder.set_min_compaction_threshold(4); - auto s = builder.build(schema_builder::compact_storage::no); - auto cf = env.make_table_for_tests(s); - auto stop_cf = deferred_stop(cf); - - // deterministic timestamp for Fri, 01 Jan 2016 00:00:00 GMT. - auto tp = db_clock::from_time_t(1451606400); - int64_t timestamp = tp.time_since_epoch().count() * 1000; // in microseconds. - - std::vector candidates; - std::unordered_set expected; - int min_threshold = cf->schema()->min_compaction_threshold(); - - const auto key = tests::generate_partition_key(s); - // add sstables that belong to same time window until min threshold is satisfied. - for (auto i = 1; i <= min_threshold; i++) { - auto sst = add_sstable_for_overlapping_test(env, cf, key.key(), key.key(), - build_stats(timestamp, timestamp, std::numeric_limits::max())); - candidates.push_back(sst); - expected.insert(sst); - } - // belongs to the time window - auto tp2 = tp + std::chrono::seconds(1800); - timestamp = tp2.time_since_epoch().count() * 1000; - auto sst = add_sstable_for_overlapping_test(env, cf, key.key(), key.key(), - build_stats(timestamp, timestamp, std::numeric_limits::max())); - candidates.push_back(sst); - expected.insert(sst); - - // doesn't belong to the time window above - auto tp3 = tp + std::chrono::seconds(4000); - timestamp = tp3.time_since_epoch().count() * 1000; - auto sst2 = add_sstable_for_overlapping_test(env, cf, key.key(), key.key(), - build_stats(timestamp, timestamp, std::numeric_limits::max())); - candidates.push_back(sst2); - - std::map options; - // Use a 1-hour time window. - options.emplace(sstring("base_time_seconds"), sstring("3600")); - - date_tiered_manifest manifest(options); - auto gc_before = gc_clock::time_point(std::chrono::seconds(0)); // disable gc before. - auto sstables = manifest.get_next_sstables(cf.as_table_state(), candidates, gc_before); - BOOST_REQUIRE(sstables.size() == size_t(min_threshold + 1)); - for (auto sst : sstables) { - BOOST_REQUIRE(expected.erase(sst)); - } - BOOST_REQUIRE(expected.empty()); - }); -} - SEASTAR_TEST_CASE(time_window_strategy_time_window_tests) { using namespace std::chrono; @@ -1814,12 +1717,14 @@ SEASTAR_TEST_CASE(sstable_expired_data_ratio) { // make sure sstable picked for tombstone compaction removal won't be promoted or demoted. BOOST_REQUIRE(descriptor.sstables.front()->get_sstable_level() == 1U); - // check tombstone compaction is disabled by default for DTCS - cs = sstables::make_compaction_strategy(sstables::compaction_strategy_type::date_tiered, {}); - descriptor = cs.get_sstables_for_compaction(cf.as_table_state(), *strategy_c, { sst }); + // check tombstone compaction is disabled by default for TWCS + auto twcs_table = env.make_table_for_tests(make_schema(sstables::compaction_strategy_type::time_window)); + auto close_twcs_table = deferred_stop(twcs_table); + cs = sstables::make_compaction_strategy(sstables::compaction_strategy_type::time_window, {}); + descriptor = cs.get_sstables_for_compaction(twcs_table.as_table_state(), *strategy_c, { sst }); BOOST_REQUIRE(descriptor.sstables.size() == 0); - cs = sstables::make_compaction_strategy(sstables::compaction_strategy_type::date_tiered, options); - descriptor = cs.get_sstables_for_compaction(cf.as_table_state(), *strategy_c, { sst }); + cs = sstables::make_compaction_strategy(sstables::compaction_strategy_type::time_window, options); + descriptor = cs.get_sstables_for_compaction(twcs_table.as_table_state(), *strategy_c, { sst }); BOOST_REQUIRE(descriptor.sstables.size() == 1); BOOST_REQUIRE(descriptor.sstables.front() == sst); diff --git a/test/lib/mutation_source_test.cc b/test/lib/mutation_source_test.cc index ff2a20b75b..8aad2a4cdf 100644 --- a/test/lib/mutation_source_test.cc +++ b/test/lib/mutation_source_test.cc @@ -968,13 +968,13 @@ void test_mutation_reader_fragments_have_monotonic_positions(tests::reader_concu }); } -static void test_date_tiered_clustering_slicing(tests::reader_concurrency_semaphore_wrapper& semaphore, populate_fn_ex populate) { +static void test_time_window_clustering_slicing(tests::reader_concurrency_semaphore_wrapper& semaphore, populate_fn_ex populate) { testlog.info(__PRETTY_FUNCTION__); simple_schema ss; auto s = schema_builder(ss.schema()) - .set_compaction_strategy(sstables::compaction_strategy_type::date_tiered) + .set_compaction_strategy(sstables::compaction_strategy_type::time_window) .build(); auto pkey = ss.make_pkey(); @@ -1601,7 +1601,7 @@ void run_mutation_reader_tests(populate_fn_ex populate, bool with_partition_rang test_range_tombstones_v2(semaphore, populate); test_reader_conversions(semaphore, populate); - test_date_tiered_clustering_slicing(semaphore, populate); + test_time_window_clustering_slicing(semaphore, populate); test_clustering_slices(semaphore, populate); test_mutation_reader_fragments_have_monotonic_positions(semaphore, populate); test_streamed_mutation_forwarding_across_range_tombstones(semaphore, populate);