From d6029a195e9d57717617d3aec951031bf18a5554 Mon Sep 17 00:00:00 2001 From: "Raphael S. Carvalho" Date: Thu, 6 Jul 2023 16:55:12 -0300 Subject: [PATCH] Remove DateTieredCompactionStrategy This is the last step of deprecation dance of DTCS. In Scylla 5.1, users were warned that DTCS was deprecated. In 5.2, altering or creation of tables with DTCS was forbidden. 5.3 branch was already created, so this is targetting 5.4. Users that refused to move away from DTCS will have Scylla falling back to the default strategy, either STCS or ICS. See: WARN 2023-07-14 09:49:11,857 [shard 0] schema_tables - Falling back to size-tiered compaction strategy after the problem: Unable to find compaction strategy class 'DateTieredCompactionStrategy Then user can later switch to a supported strategy with alter table. Signed-off-by: Raphael S. Carvalho Closes #14559 --- compaction/compaction_strategy.cc | 193 ------------ compaction/compaction_strategy.hh | 4 - compaction/compaction_strategy_type.hh | 1 - compaction/date_tiered_compaction_strategy.hh | 277 ------------------ cql3/statements/create_table_statement.cc | 18 -- db/config.cc | 2 +- test/boost/sstable_compaction_test.cc | 109 +------ test/lib/mutation_source_test.cc | 6 +- 8 files changed, 11 insertions(+), 599 deletions(-) delete mode 100644 compaction/date_tiered_compaction_strategy.hh diff --git a/compaction/compaction_strategy.cc b/compaction/compaction_strategy.cc index 9be7910184..a86db5cd24 100644 --- a/compaction/compaction_strategy.cc +++ b/compaction/compaction_strategy.cc @@ -24,7 +24,6 @@ #include #include #include "size_tiered_compaction_strategy.hh" -#include "date_tiered_compaction_strategy.hh" #include "leveled_compaction_strategy.hh" #include "time_window_compaction_strategy.hh" #include "backlog_controller.hh" @@ -32,7 +31,6 @@ #include "size_tiered_backlog_tracker.hh" #include "leveled_manifest.hh" -logging::logger date_tiered_manifest::logger = logging::logger("DateTieredCompactionStrategy"); logging::logger leveled_manifest::logger("LeveledManifest"); namespace sstables { @@ -499,195 +497,8 @@ std::unique_ptr time_window_compaction_strateg } // namespace sstables -std::vector -date_tiered_manifest::get_next_sstables(table_state& table_s, std::vector& uncompacting, gc_clock::time_point compaction_time) { - if (table_s.main_sstable_set().all()->empty()) { - return {}; - } - - // Find fully expired SSTables. Those will be included no matter what. - auto expired = table_s.fully_expired_sstables(uncompacting, compaction_time); - - if (!expired.empty()) { - auto is_expired = [&] (const sstables::shared_sstable& s) { return expired.contains(s); }; - uncompacting.erase(boost::remove_if(uncompacting, is_expired), uncompacting.end()); - } - - auto compaction_candidates = get_next_non_expired_sstables(table_s, uncompacting, compaction_time); - if (!expired.empty()) { - compaction_candidates.insert(compaction_candidates.end(), expired.begin(), expired.end()); - } - return compaction_candidates; -} - -int64_t date_tiered_manifest::get_estimated_tasks(table_state& table_s) const { - int base = table_s.schema()->min_compaction_threshold(); - int64_t now = get_now(table_s.main_sstable_set().all()); - std::vector sstables; - int64_t n = 0; - - auto all_sstables = table_s.main_sstable_set().all(); - sstables.reserve(all_sstables->size()); - for (auto& entry : *all_sstables) { - sstables.push_back(entry); - } - auto candidates = filter_old_sstables(sstables, _options.max_sstable_age, now); - auto buckets = get_buckets(create_sst_and_min_timestamp_pairs(candidates), _options.base_time, base, now); - - for (auto& bucket : buckets) { - if (bucket.size() >= size_t(table_s.schema()->min_compaction_threshold())) { - n += std::ceil(double(bucket.size()) / table_s.schema()->max_compaction_threshold()); - } - } - return n; -} - -std::vector -date_tiered_manifest::get_next_non_expired_sstables(table_state& table_s, std::vector& non_expiring_sstables, gc_clock::time_point compaction_time) { - int base = table_s.schema()->min_compaction_threshold(); - int64_t now = get_now(table_s.main_sstable_set().all()); - auto most_interesting = get_compaction_candidates(table_s, non_expiring_sstables, now, base); - - return most_interesting; - - // FIXME: implement functionality below that will look for a single sstable with worth dropping tombstone, - // iff strategy didn't find anything to compact. So it's not essential. -#if 0 - // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone - // ratio is greater than threshold. - - List sstablesWithTombstones = Lists.newArrayList(); - for (SSTableReader sstable : nonExpiringSSTables) - { - if (worthDroppingTombstones(sstable, gcBefore)) - sstablesWithTombstones.add(sstable); - } - if (sstablesWithTombstones.isEmpty()) - return Collections.emptyList(); - - return Collections.singletonList(Collections.min(sstablesWithTombstones, new SSTableReader.SizeComparator())); -#endif -} - -std::vector -date_tiered_manifest::get_compaction_candidates(table_state& table_s, std::vector candidate_sstables, int64_t now, int base) { - int min_threshold = table_s.schema()->min_compaction_threshold(); - int max_threshold = table_s.schema()->max_compaction_threshold(); - auto candidates = filter_old_sstables(candidate_sstables, _options.max_sstable_age, now); - - auto buckets = get_buckets(create_sst_and_min_timestamp_pairs(candidates), _options.base_time, base, now); - - return newest_bucket(buckets, min_threshold, max_threshold, now, _options.base_time); -} - -int64_t date_tiered_manifest::get_now(lw_shared_ptr shared_set) { - int64_t max_timestamp = 0; - for (auto& sst : *shared_set) { - int64_t candidate = sst->get_stats_metadata().max_timestamp; - max_timestamp = candidate > max_timestamp ? candidate : max_timestamp; - } - return max_timestamp; -} - -std::vector -date_tiered_manifest::filter_old_sstables(std::vector sstables, api::timestamp_type max_sstable_age, int64_t now) { - if (max_sstable_age == 0) { - return sstables; - } - int64_t cutoff = now - max_sstable_age; - - std::erase_if(sstables, [cutoff] (auto& sst) { - return sst->get_stats_metadata().max_timestamp < cutoff; - }); - - return sstables; -} - -std::vector> -date_tiered_manifest::create_sst_and_min_timestamp_pairs(const std::vector& sstables) { - std::vector> sstable_min_timestamp_pairs; - sstable_min_timestamp_pairs.reserve(sstables.size()); - for (auto& sst : sstables) { - sstable_min_timestamp_pairs.emplace_back(sst, sst->get_stats_metadata().min_timestamp); - } - return sstable_min_timestamp_pairs; -} - -date_tiered_compaction_strategy_options::date_tiered_compaction_strategy_options(const std::map& options) { - using namespace cql3::statements; - - auto tmp_value = sstables::compaction_strategy_impl::get_value(options, TIMESTAMP_RESOLUTION_KEY); - auto target_unit = tmp_value ? tmp_value.value() : DEFAULT_TIMESTAMP_RESOLUTION; - - tmp_value = sstables::compaction_strategy_impl::get_value(options, MAX_SSTABLE_AGE_KEY); - auto fractional_days = property_definitions::to_double(MAX_SSTABLE_AGE_KEY, tmp_value, DEFAULT_MAX_SSTABLE_AGE_DAYS); - int64_t max_sstable_age_in_hours = std::lround(fractional_days * 24); - max_sstable_age = duration_conversor::convert(target_unit, std::chrono::hours(max_sstable_age_in_hours)); - - tmp_value = sstables::compaction_strategy_impl::get_value(options, BASE_TIME_KEY); - auto base_time_seconds = property_definitions::to_long(BASE_TIME_KEY, tmp_value, DEFAULT_BASE_TIME_SECONDS); - base_time = duration_conversor::convert(target_unit, std::chrono::seconds(base_time_seconds)); -} - -date_tiered_compaction_strategy_options::date_tiered_compaction_strategy_options() { - auto max_sstable_age_in_hours = int64_t(DEFAULT_MAX_SSTABLE_AGE_DAYS * 24); - max_sstable_age = std::chrono::duration_cast(std::chrono::hours(max_sstable_age_in_hours)).count(); - base_time = std::chrono::duration_cast(std::chrono::seconds(DEFAULT_BASE_TIME_SECONDS)).count(); -} - namespace sstables { -date_tiered_compaction_strategy::date_tiered_compaction_strategy(const std::map& options) - : compaction_strategy_impl(options) - , _manifest(options) -{ - clogger.warn("DateTieredCompactionStrategy is deprecated. Usually cases for which it is used are better handled by TimeWindowCompactionStrategy." - " Please change your compaction strategy to TWCS as DTCS will be retired in the near future"); - - // tombstone compaction is disabled by default because: - // - deletion shouldn't be used with DTCS; rather data is deleted through TTL. - // - with time series workloads, it's usually better to wait for whole sstable to be expired rather than - // compacting a single sstable when it's more than 20% (default value) expired. - // For more details, see CASSANDRA-9234 - if (!options.contains(TOMBSTONE_COMPACTION_INTERVAL_OPTION) && !options.contains(TOMBSTONE_THRESHOLD_OPTION)) { - _disable_tombstone_compaction = true; - date_tiered_manifest::logger.debug("Disabling tombstone compactions for DTCS"); - } else { - date_tiered_manifest::logger.debug("Enabling tombstone compactions for DTCS"); - } - - _use_clustering_key_filter = true; -} - -compaction_descriptor date_tiered_compaction_strategy::get_sstables_for_compaction(table_state& table_s, strategy_control& control, std::vector candidates) { - auto compaction_time = gc_clock::now(); - auto sstables = _manifest.get_next_sstables(table_s, candidates, compaction_time); - - if (!sstables.empty()) { - date_tiered_manifest::logger.debug("datetiered: Compacting {} out of {} sstables", sstables.size(), candidates.size()); - return sstables::compaction_descriptor(std::move(sstables)); - } - - // filter out sstables which droppable tombstone ratio isn't greater than the defined threshold. - auto e = boost::range::remove_if(candidates, [this, compaction_time, &table_s] (const sstables::shared_sstable& sst) -> bool { - return !worth_dropping_tombstones(sst, compaction_time, table_s.get_tombstone_gc_state()); - }); - candidates.erase(e, candidates.end()); - if (candidates.empty()) { - return sstables::compaction_descriptor(); - } - // find oldest sstable which is worth dropping tombstones because they are more unlikely to - // shadow data from other sstables, and it also tends to be relatively big. - auto it = std::min_element(candidates.begin(), candidates.end(), [] (auto& i, auto& j) { - return i->get_stats_metadata().min_timestamp < j->get_stats_metadata().min_timestamp; - }); - return sstables::compaction_descriptor({ *it }); -} - -std::unique_ptr date_tiered_compaction_strategy::make_backlog_tracker() const { - return std::make_unique(); -} - size_tiered_compaction_strategy::size_tiered_compaction_strategy(const std::map& options) : compaction_strategy_impl(options) , _options(options) @@ -775,9 +586,6 @@ compaction_strategy make_compaction_strategy(compaction_strategy_type strategy, case compaction_strategy_type::leveled: impl = ::make_shared(options); break; - case compaction_strategy_type::date_tiered: - impl = ::make_shared(options); - break; case compaction_strategy_type::time_window: impl = ::make_shared(options); break; @@ -796,7 +604,6 @@ compaction_strategy_state compaction_strategy_state::make(const compaction_strat switch (cs.type()) { case compaction_strategy_type::null: case compaction_strategy_type::size_tiered: - case compaction_strategy_type::date_tiered: return compaction_strategy_state(default_empty_state{}); case compaction_strategy_type::leveled: return compaction_strategy_state(leveled_compaction_strategy_state{}); diff --git a/compaction/compaction_strategy.hh b/compaction/compaction_strategy.hh index b3e37d5c42..212f59b5a8 100644 --- a/compaction/compaction_strategy.hh +++ b/compaction/compaction_strategy.hh @@ -71,8 +71,6 @@ public: return "SizeTieredCompactionStrategy"; case compaction_strategy_type::leveled: return "LeveledCompactionStrategy"; - case compaction_strategy_type::date_tiered: - return "DateTieredCompactionStrategy"; case compaction_strategy_type::time_window: return "TimeWindowCompactionStrategy"; default: @@ -89,8 +87,6 @@ public: return compaction_strategy_type::size_tiered; } else if (short_name == "LeveledCompactionStrategy") { return compaction_strategy_type::leveled; - } else if (short_name == "DateTieredCompactionStrategy") { - return compaction_strategy_type::date_tiered; } else if (short_name == "TimeWindowCompactionStrategy") { return compaction_strategy_type::time_window; } else { diff --git a/compaction/compaction_strategy_type.hh b/compaction/compaction_strategy_type.hh index 73fa4941c6..6da2e75953 100644 --- a/compaction/compaction_strategy_type.hh +++ b/compaction/compaction_strategy_type.hh @@ -14,7 +14,6 @@ enum class compaction_strategy_type { null, size_tiered, leveled, - date_tiered, time_window, }; diff --git a/compaction/date_tiered_compaction_strategy.hh b/compaction/date_tiered_compaction_strategy.hh deleted file mode 100644 index 7dad7b0ba4..0000000000 --- a/compaction/date_tiered_compaction_strategy.hh +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Copyright (C) 2016-present-2017 ScyllaDB - * - * Modified by ScyllaDB - */ - -/* - * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0) - */ - -#pragma once - -#include -#include -#include -#include -#include -#include "sstables/sstables.hh" -#include "compaction.hh" -#include "timestamp.hh" -#include "cql3/statements/property_definitions.hh" -#include "compaction_strategy_impl.hh" - -static constexpr double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365; -static constexpr int64_t DEFAULT_BASE_TIME_SECONDS = 60; - -struct duration_conversor { - // Convert given duration to TargetDuration and return value as timestamp. - template - static api::timestamp_type convert(SourceDuration d) { - return std::chrono::duration_cast(d).count(); - } - - // Convert given duration to duration that is represented by the string - // target_duration, and return value as timestamp. - template - static api::timestamp_type convert(const sstring& target_duration, SourceDuration d) { - if (target_duration == "HOURS") { - return convert(d); - } else if (target_duration == "MICROSECONDS") { - return convert(d); - } else if (target_duration == "MILLISECONDS") { - return convert(d); - } else if (target_duration == "MINUTES") { - return convert(d); - } else if (target_duration == "NANOSECONDS") { - return convert(d); - } else if (target_duration == "SECONDS") { - return convert(d); - } else { - throw std::runtime_error(format("target duration {} is not available", target_duration)); - } - } -}; - -class date_tiered_compaction_strategy_options { - const sstring DEFAULT_TIMESTAMP_RESOLUTION = "MICROSECONDS"; - const sstring TIMESTAMP_RESOLUTION_KEY = "timestamp_resolution"; - const sstring MAX_SSTABLE_AGE_KEY = "max_sstable_age_days"; - const sstring BASE_TIME_KEY = "base_time_seconds"; - - api::timestamp_type max_sstable_age; - api::timestamp_type base_time; -public: - date_tiered_compaction_strategy_options(const std::map& options); - - date_tiered_compaction_strategy_options(); -private: - - friend class date_tiered_manifest; -}; - -class date_tiered_manifest { - date_tiered_compaction_strategy_options _options; -public: - static logging::logger logger; - - date_tiered_manifest() = delete; - - date_tiered_manifest(const std::map& options) - : _options(options) {} - - std::vector - get_next_sstables(table_state& table_s, std::vector& uncompacting, gc_clock::time_point compaction_time); - - int64_t get_estimated_tasks(table_state& table_s) const; -private: - std::vector - get_next_non_expired_sstables(table_state& table_s, std::vector& non_expiring_sstables, gc_clock::time_point compaction_time); - - std::vector - get_compaction_candidates(table_state& table_s, std::vector candidate_sstables, int64_t now, int base); - - /** - * Gets the timestamp that DateTieredCompactionStrategy considers to be the "current time". - * @return the maximum timestamp across all SSTables. - */ - static int64_t get_now(lw_shared_ptr shared_set); - - /** - * Removes all sstables with max timestamp older than maxSSTableAge. - * @return a list of sstables with the oldest sstables excluded - */ - static std::vector - filter_old_sstables(std::vector sstables, api::timestamp_type max_sstable_age, int64_t now); - - /** - * - * @param sstables - * @return - */ - static std::vector> - create_sst_and_min_timestamp_pairs(const std::vector& sstables); - - /** - * A target time span used for bucketing SSTables based on timestamps. - */ - struct target { - // How big a range of timestamps fit inside the target. - int64_t size; - // A timestamp t hits the target iff t / size == divPosition. - int64_t div_position; - - target() = delete; - target(int64_t size, int64_t div_position) : size(size), div_position(div_position) {} - - /** - * Compares the target to a timestamp. - * @param timestamp the timestamp to compare. - * @return a negative integer, zero, or a positive integer as the target lies before, covering, or after than the timestamp. - */ - int compare_to_timestamp(int64_t timestamp) { - auto ts1 = div_position; - auto ts2 = timestamp / size; - return (ts1 > ts2 ? 1 : (ts1 == ts2 ? 0 : -1)); - } - - /** - * Tells if the timestamp hits the target. - * @param timestamp the timestamp to test. - * @return true iff timestamp / size == divPosition. - */ - bool on_target(int64_t timestamp) { - return compare_to_timestamp(timestamp) == 0; - } - - /** - * Gets the next target, which represents an earlier time span. - * @param base The number of contiguous targets that will have the same size. Targets following those will be base times as big. - * @return - */ - target next_target(int base) - { - if (div_position % base > 0) { - return target(size, div_position - 1); - } else { - return target(size * base, div_position / base - 1); - } - } - }; - - - /** - * Group files with similar min timestamp into buckets. Files with recent min timestamps are grouped together into - * buckets designated to short timespans while files with older timestamps are grouped into buckets representing - * longer timespans. - * @param files pairs consisting of a file and its min timestamp - * @param timeUnit - * @param base - * @param now - * @return a list of buckets of files. The list is ordered such that the files with newest timestamps come first. - * Each bucket is also a list of files ordered from newest to oldest. - */ - std::vector> - get_buckets(std::vector>&& files, api::timestamp_type time_unit, int base, int64_t now) const { - // Sort files by age. Newest first. - std::sort(files.begin(), files.end(), [] (auto& i, auto& j) { - return i.second > j.second; - }); - - std::vector> buckets; - auto target = get_initial_target(now, time_unit); - auto it = files.begin(); - - while (it != files.end()) { - bool finish = false; - while (!target.on_target(it->second)) { - // If the file is too new for the target, skip it. - if (target.compare_to_timestamp(it->second) < 0) { - it++; - if (it == files.end()) { - finish = true; - break; - } - } else { // If the file is too old for the target, switch targets. - target = target.next_target(base); - } - } - if (finish) { - break; - } - - std::vector bucket; - while (target.on_target(it->second)) { - bucket.push_back(it->first); - it++; - if (it == files.end()) { - break; - } - } - buckets.push_back(bucket); - } - - return buckets; - } - - target get_initial_target(uint64_t now, int64_t time_unit) const { - return target(time_unit, now / time_unit); - } - - /** - * @param buckets list of buckets, sorted from newest to oldest, from which to return the newest bucket within thresholds. - * @param minThreshold minimum number of sstables in a bucket to qualify. - * @param maxThreshold maximum number of sstables to compact at once (the returned bucket will be trimmed down to this). - * @return a bucket (list) of sstables to compact. - */ - std::vector - newest_bucket(std::vector>& buckets, int min_threshold, int max_threshold, - int64_t now, api::timestamp_type base_time) { - - // If the "incoming window" has at least minThreshold SSTables, choose that one. - // For any other bucket, at least 2 SSTables is enough. - // In any case, limit to maxThreshold SSTables. - target incoming_window = get_initial_target(now, base_time); - for (auto& bucket : buckets) { - auto min_timestamp = bucket.front()->get_stats_metadata().min_timestamp; - if (bucket.size() >= size_t(min_threshold) || - (bucket.size() >= 2 && !incoming_window.on_target(min_timestamp))) { - trim_to_threshold(bucket, max_threshold); - return bucket; - } - } - return {}; - } - - - /** - * @param bucket list of sstables, ordered from newest to oldest by getMinTimestamp(). - * @param maxThreshold maximum number of sstables in a single compaction task. - * @return A bucket trimmed to the maxThreshold newest sstables. - */ - static void trim_to_threshold(std::vector& bucket, int max_threshold) { - // Trim the oldest sstables off the end to meet the maxThreshold - bucket.resize(std::min(bucket.size(), size_t(max_threshold))); - } -}; - -namespace sstables { - -class date_tiered_compaction_strategy : public compaction_strategy_impl { - date_tiered_manifest _manifest; -public: - date_tiered_compaction_strategy(const std::map& options); - virtual compaction_descriptor get_sstables_for_compaction(table_state& table_s, strategy_control& control, std::vector candidates) override; - - virtual int64_t estimated_pending_compactions(table_state& table_s) const override { - return _manifest.get_estimated_tasks(table_s); - } - - virtual compaction_strategy_type type() const override { - return compaction_strategy_type::date_tiered; - } - - virtual std::unique_ptr make_backlog_tracker() const override; -}; - -} diff --git a/cql3/statements/create_table_statement.cc b/cql3/statements/create_table_statement.cc index ea409ab2f6..e1f9e5b654 100644 --- a/cql3/statements/create_table_statement.cc +++ b/cql3/statements/create_table_statement.cc @@ -445,24 +445,6 @@ std::optional check_restricted_table_properties( // Evaluate whether the strategy to evaluate was explicitly passed auto cs = (strategy) ? strategy : current_strategy; - if (strategy && *strategy == sstables::compaction_strategy_type::date_tiered) { - switch(qp.db().get_config().restrict_dtcs()) { - case db::tri_mode_restriction_t::mode::TRUE: - throw exceptions::configuration_exception( - "DateTieredCompactionStrategy is deprecated, and " - "forbidden by the current configuration. Please use " - "TimeWindowCompactionStrategy instead. You may also override this " - "restriction by setting the restrict_dtcs configuration option " - "to false."); - case db::tri_mode_restriction_t::mode::WARN: - return format("DateTieredCompactionStrategy is deprecated, " - "but was used for table {}.{}. The restrict_dtcs " - "configuration option can be changed to silence this warning " - " or make it into an error.", keyspace, table); - case db::tri_mode_restriction_t::mode::FALSE: - break; - } - } if (cs == sstables::compaction_strategy_type::time_window) { std::map options = (strategy) ? cfprops.get_compaction_type_options() : (*schema)->compaction_strategy_options(); sstables::time_window_compaction_strategy_options twcs_options(options); diff --git a/db/config.cc b/db/config.cc index 9783722d27..0b47655d56 100644 --- a/db/config.cc +++ b/db/config.cc @@ -983,7 +983,7 @@ db::config::config(std::shared_ptr exts) , flush_schema_tables_after_modification(this, "flush_schema_tables_after_modification", liveness::LiveUpdate, value_status::Used, true, "Flush tables in the system_schema keyspace after schema modification. This is required for crash recovery, but slows down tests and can be disabled for them") , restrict_replication_simplestrategy(this, "restrict_replication_simplestrategy", liveness::LiveUpdate, value_status::Used, db::tri_mode_restriction_t::mode::FALSE, "Controls whether to disable SimpleStrategy replication. Can be true, false, or warn.") - , restrict_dtcs(this, "restrict_dtcs", liveness::LiveUpdate, value_status::Used, db::tri_mode_restriction_t::mode::TRUE, "Controls whether to prevent setting DateTieredCompactionStrategy. Can be true, false, or warn.") + , restrict_dtcs(this, "restrict_dtcs", liveness::LiveUpdate, value_status::Unused, db::tri_mode_restriction_t::mode::TRUE, "Controls whether to prevent setting DateTieredCompactionStrategy. Can be true, false, or warn.") , restrict_twcs_without_default_ttl(this, "restrict_twcs_without_default_ttl", liveness::LiveUpdate, value_status::Used, db::tri_mode_restriction_t::mode::WARN, "Controls whether to prevent creating TimeWindowCompactionStrategy tables without a default TTL. Can be true, false, or warn.") , restrict_future_timestamp(this, "restrict_future_timestamp",liveness::LiveUpdate, value_status::Used, true, "Controls whether to detect and forbid unreasonable USING TIMESTAMP, more than 3 days into the future.") , ignore_truncation_record(this, "unsafe_ignore_truncation_record", value_status::Used, false, diff --git a/test/boost/sstable_compaction_test.cc b/test/boost/sstable_compaction_test.cc index 33dc8fa40a..224a5a109c 100644 --- a/test/boost/sstable_compaction_test.cc +++ b/test/boost/sstable_compaction_test.cc @@ -37,7 +37,6 @@ #include "range.hh" #include "partition_slice_builder.hh" #include "compaction/compaction_strategy_impl.hh" -#include "compaction/date_tiered_compaction_strategy.hh" #include "compaction/time_window_compaction_strategy.hh" #include "compaction/leveled_compaction_strategy.hh" #include "test/lib/mutation_assertions.hh" @@ -1312,102 +1311,6 @@ SEASTAR_TEST_CASE(compaction_with_fully_expired_table) { }); } -SEASTAR_TEST_CASE(basic_date_tiered_strategy_test) { - return test_env::do_with_async([] (test_env& env) { - schema_builder builder(make_shared_schema({}, some_keyspace, some_column_family, - {{"p1", utf8_type}}, {}, {}, {}, utf8_type)); - builder.set_min_compaction_threshold(4); - auto s = builder.build(schema_builder::compact_storage::no); - auto cf = env.make_table_for_tests(s); - auto stop_cf = deferred_stop(cf); - - std::vector candidates; - std::unordered_set expected; - int min_threshold = cf->schema()->min_compaction_threshold(); - auto now = db_clock::now(); - auto past_hour = now - std::chrono::seconds(3600); - int64_t timestamp_for_now = now.time_since_epoch().count() * 1000; - int64_t timestamp_for_past_hour = past_hour.time_since_epoch().count() * 1000; - - const auto key = tests::generate_partition_key(s); - for (auto i = 1; i <= min_threshold; i++) { - auto sst = add_sstable_for_overlapping_test(env, cf, key.key(), key.key(), - build_stats(timestamp_for_now, timestamp_for_now, std::numeric_limits::max())); - candidates.push_back(sst); - expected.insert(sst); - } - // add sstable that belong to a different time tier. - auto sst = add_sstable_for_overlapping_test(env, cf, key.key(), key.key(), - build_stats(timestamp_for_past_hour, timestamp_for_past_hour, std::numeric_limits::max())); - candidates.push_back(sst); - - auto gc_before = gc_clock::now() - cf->schema()->gc_grace_seconds(); - std::map options; - date_tiered_manifest manifest(options); - auto sstables = manifest.get_next_sstables(cf.as_table_state(), candidates, gc_before); - BOOST_REQUIRE(sstables.size() == 4); - for (auto& sst : sstables) { - BOOST_REQUIRE(expected.erase(sst)); - } - BOOST_REQUIRE(expected.empty()); - }); -} - -SEASTAR_TEST_CASE(date_tiered_strategy_test_2) { - return test_env::do_with_async([] (test_env& env) { - schema_builder builder(make_shared_schema({}, some_keyspace, some_column_family, - {{"p1", utf8_type}}, {}, {}, {}, utf8_type)); - builder.set_min_compaction_threshold(4); - auto s = builder.build(schema_builder::compact_storage::no); - auto cf = env.make_table_for_tests(s); - auto stop_cf = deferred_stop(cf); - - // deterministic timestamp for Fri, 01 Jan 2016 00:00:00 GMT. - auto tp = db_clock::from_time_t(1451606400); - int64_t timestamp = tp.time_since_epoch().count() * 1000; // in microseconds. - - std::vector candidates; - std::unordered_set expected; - int min_threshold = cf->schema()->min_compaction_threshold(); - - const auto key = tests::generate_partition_key(s); - // add sstables that belong to same time window until min threshold is satisfied. - for (auto i = 1; i <= min_threshold; i++) { - auto sst = add_sstable_for_overlapping_test(env, cf, key.key(), key.key(), - build_stats(timestamp, timestamp, std::numeric_limits::max())); - candidates.push_back(sst); - expected.insert(sst); - } - // belongs to the time window - auto tp2 = tp + std::chrono::seconds(1800); - timestamp = tp2.time_since_epoch().count() * 1000; - auto sst = add_sstable_for_overlapping_test(env, cf, key.key(), key.key(), - build_stats(timestamp, timestamp, std::numeric_limits::max())); - candidates.push_back(sst); - expected.insert(sst); - - // doesn't belong to the time window above - auto tp3 = tp + std::chrono::seconds(4000); - timestamp = tp3.time_since_epoch().count() * 1000; - auto sst2 = add_sstable_for_overlapping_test(env, cf, key.key(), key.key(), - build_stats(timestamp, timestamp, std::numeric_limits::max())); - candidates.push_back(sst2); - - std::map options; - // Use a 1-hour time window. - options.emplace(sstring("base_time_seconds"), sstring("3600")); - - date_tiered_manifest manifest(options); - auto gc_before = gc_clock::time_point(std::chrono::seconds(0)); // disable gc before. - auto sstables = manifest.get_next_sstables(cf.as_table_state(), candidates, gc_before); - BOOST_REQUIRE(sstables.size() == size_t(min_threshold + 1)); - for (auto sst : sstables) { - BOOST_REQUIRE(expected.erase(sst)); - } - BOOST_REQUIRE(expected.empty()); - }); -} - SEASTAR_TEST_CASE(time_window_strategy_time_window_tests) { using namespace std::chrono; @@ -1814,12 +1717,14 @@ SEASTAR_TEST_CASE(sstable_expired_data_ratio) { // make sure sstable picked for tombstone compaction removal won't be promoted or demoted. BOOST_REQUIRE(descriptor.sstables.front()->get_sstable_level() == 1U); - // check tombstone compaction is disabled by default for DTCS - cs = sstables::make_compaction_strategy(sstables::compaction_strategy_type::date_tiered, {}); - descriptor = cs.get_sstables_for_compaction(cf.as_table_state(), *strategy_c, { sst }); + // check tombstone compaction is disabled by default for TWCS + auto twcs_table = env.make_table_for_tests(make_schema(sstables::compaction_strategy_type::time_window)); + auto close_twcs_table = deferred_stop(twcs_table); + cs = sstables::make_compaction_strategy(sstables::compaction_strategy_type::time_window, {}); + descriptor = cs.get_sstables_for_compaction(twcs_table.as_table_state(), *strategy_c, { sst }); BOOST_REQUIRE(descriptor.sstables.size() == 0); - cs = sstables::make_compaction_strategy(sstables::compaction_strategy_type::date_tiered, options); - descriptor = cs.get_sstables_for_compaction(cf.as_table_state(), *strategy_c, { sst }); + cs = sstables::make_compaction_strategy(sstables::compaction_strategy_type::time_window, options); + descriptor = cs.get_sstables_for_compaction(twcs_table.as_table_state(), *strategy_c, { sst }); BOOST_REQUIRE(descriptor.sstables.size() == 1); BOOST_REQUIRE(descriptor.sstables.front() == sst); diff --git a/test/lib/mutation_source_test.cc b/test/lib/mutation_source_test.cc index ff2a20b75b..8aad2a4cdf 100644 --- a/test/lib/mutation_source_test.cc +++ b/test/lib/mutation_source_test.cc @@ -968,13 +968,13 @@ void test_mutation_reader_fragments_have_monotonic_positions(tests::reader_concu }); } -static void test_date_tiered_clustering_slicing(tests::reader_concurrency_semaphore_wrapper& semaphore, populate_fn_ex populate) { +static void test_time_window_clustering_slicing(tests::reader_concurrency_semaphore_wrapper& semaphore, populate_fn_ex populate) { testlog.info(__PRETTY_FUNCTION__); simple_schema ss; auto s = schema_builder(ss.schema()) - .set_compaction_strategy(sstables::compaction_strategy_type::date_tiered) + .set_compaction_strategy(sstables::compaction_strategy_type::time_window) .build(); auto pkey = ss.make_pkey(); @@ -1601,7 +1601,7 @@ void run_mutation_reader_tests(populate_fn_ex populate, bool with_partition_rang test_range_tombstones_v2(semaphore, populate); test_reader_conversions(semaphore, populate); - test_date_tiered_clustering_slicing(semaphore, populate); + test_time_window_clustering_slicing(semaphore, populate); test_clustering_slices(semaphore, populate); test_mutation_reader_fragments_have_monotonic_positions(semaphore, populate); test_streamed_mutation_forwarding_across_range_tombstones(semaphore, populate);