Add computed columns

Merged patch series by Piotr Sarna:

This series introduces the concept of "computed" column, which represents
values not provided directly by the user, but computed on the fly -
possibly using other column values. It will be used in the future to
implement map value indexing, collection indexing, etc. Right now the only
use is the token column for secondary indexes - which is a column computed
from the base partition key value.

After this series, another one that depends on it and adds map value
indexing will be pushed.

Tests: unit(dev)

Piotr Sarna (14):
  schema: add computed info to column definition
  schema: add implementation of computing token column
  schema: allow marking columns as computed in schema builder
  service: add computed columns feature
  view: check for computed columns in view
  view: remove unused token_for function
  database: add fixing previous secondary index schemas
  tests: disable computed columns feature in schema change test
  tests: add schema change test regeneration comment
  db: add system_schema.computed_columns
  docs: init system_schema_keyspace.md with column computations
  tests: generate new test case for schema change + computed cols
  index: mark token column as 'computed' when creating mv
  tests: add checking computed columns in SI

 column_computation.hh                         |  63 ++++++++
 db/schema_features.hh                         |   4 +-
 db/schema_tables.hh                           |   4 +
 idl/frozen_schema.idl.hh                      |   1 +
 schema.hh                                     |  40 +++++
 schema_builder.hh                             |   4 +-
 schema_mutations.hh                           |  18 ++-
 service/storage_service.hh                    |   8 +
 view_info.hh                                  |   2 -
 database.cc                                   |   6 +-
 db/schema_tables.cc                           | 146 ++++++++++++++++--
 db/view/view.cc                               |  46 +++---
 index/secondary_index_manager.cc              |   2 +-
 schema.cc                                     |  58 ++++++-
 schema_mutations.cc                           |  14 +-
 service/storage_service.cc                    |   5 +
 tests/schema_change_test.cc                   |  63 ++++++--
 tests/secondary_index_test.cc                 |  28 ++++
 docs/system_schema_keyspace.md                |  40 +++++

 plus about 200 new test sstable files

column_computation.hh

@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2019 ScyllaDB
+ */
+
+/*
+ * This file is part of Scylla.
+ *
+ * Scylla is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Scylla is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "bytes.hh"
+
+class schema;
+class partition_key;
+class clustering_row;
+
+class column_computation;
+using column_computation_ptr = std::unique_ptr<column_computation>;
+
+/*
+ * Column computation represents a computation performed in order to obtain a value for a computed column.
+ * Computed columns description is also available at docs/system_schema_keyspace.md. They hold values
+ * not provided directly by the user, but rather computed: from other column values and possibly other sources.
+ * This class is able to serialize/deserialize column computations and perform the computation itself,
+ * based on given schema, partition key and clustering row. Responsibility for providing enough data
+ * in the clustering row in order for computation to succeed belongs to the caller. In particular,
+ * generating a value might involve performing a read-before-write if the computation is performed
+ * on more values than are present in the update request.
+ */
+class column_computation {
+public:
+    virtual ~column_computation() = default;
+
+    static column_computation_ptr deserialize(bytes_view raw);
+    static column_computation_ptr deserialize(const Json::Value& json);
+
+    virtual column_computation_ptr clone() const = 0;
+
+    virtual bytes serialize() const = 0;
+    virtual bytes_opt compute_value(const schema& schema, const partition_key& key, const clustering_row& row) const = 0;
+};
+
+class token_column_computation : public column_computation {
+public:
+    virtual column_computation_ptr clone() const override {
+        return std::make_unique<token_column_computation>(*this);
+    }
+    virtual bytes serialize() const override;
+    virtual bytes_opt compute_value(const schema& schema, const partition_key& key, const clustering_row& row) const override;
+};

database.cc

@@ -90,6 +90,8 @@
 #include "user_types_metadata.hh"
 #include <seastar/core/shared_ptr_incomplete.hh>
 
+#include "schema_builder.hh"
+
 using namespace std::chrono_literals;
 using namespace db;
 
@@ -600,7 +602,9 @@ future<> database::parse_system_tables(distributed<service::storage_proxy>& prox
         return do_parse_schema_tables(proxy, db::schema_tables::VIEWS, [this, &proxy] (schema_result_value_type &v) {
             return create_views_from_schema_partition(proxy, v.second).then([this] (std::vector<view_ptr> views) {
                 return parallel_for_each(views.begin(), views.end(), [this] (auto&& v) {
-                    return this->add_column_family_and_make_directory(v);
+                    return this->add_column_family_and_make_directory(v).then([this, v] {
+                        return maybe_update_legacy_secondary_index_mv_schema(*this, v);
+                    });
                 });
             });
         });

db/schema_features.hh

@@ -32,11 +32,13 @@ enum class schema_feature {
     // tombstones in an empty partition expire.
     // See https://github.com/scylladb/scylla/issues/4485
     DIGEST_INSENSITIVE_TO_EXPIRY,
+    COMPUTED_COLUMNS,
 };
 
 using schema_features = enum_set<super_enum<schema_feature,
     schema_feature::VIEW_VIRTUAL_COLUMNS,
-    schema_feature::DIGEST_INSENSITIVE_TO_EXPIRY
+    schema_feature::DIGEST_INSENSITIVE_TO_EXPIRY,
+    schema_feature::COMPUTED_COLUMNS
     >>;
 
 }

db/schema_tables.cc

@@ -84,6 +84,7 @@
 #include "user_types_metadata.hh"
 
 #include "index/target_parser.hh"
+#include "service/storage_service.hh"
 
 using namespace db::system_keyspace;
 using namespace std::chrono_literals;
@@ -155,9 +156,12 @@ struct user_types_to_drop final {
 
 static future<> do_merge_schema(distributed<service::storage_proxy>&, std::vector<mutation>, bool do_flush);
 
+using computed_columns_map = std::unordered_map<bytes, column_computation_ptr>;
+static computed_columns_map get_computed_columns(const schema_mutations& sm);
+
 static std::vector<column_definition> create_columns_from_column_rows(
                 const query::result_set& rows, const sstring& keyspace,
-                const sstring& table, bool is_super, column_view_virtual is_view_virtual);
+                const sstring& table, bool is_super, column_view_virtual is_view_virtual, const computed_columns_map& computed_columns);
 
 
 static std::vector<index_metadata> create_indices_from_index_rows(const query::result_set& rows,
@@ -171,6 +175,9 @@ static index_metadata create_index_from_index_row(const query::result_set_row& r
 static void add_column_to_schema_mutation(schema_ptr, const column_definition&,
                 api::timestamp_type, mutation&);
 
+static void add_computed_column_to_schema_mutation(schema_ptr, const column_definition&,
+                api::timestamp_type, mutation&);
+
 static void add_index_to_schema_mutation(schema_ptr table,
                 const index_metadata& index, api::timestamp_type timestamp,
                 mutation& mutation);
@@ -345,6 +352,38 @@ schema_ptr view_virtual_columns() {
     return schema;
 }
 
+// Computed columns are a special kind of columns. Rather than having their value provided directly
+// by the user, they are computed - possibly from other column values. This table stores which columns
+// for a given table are computed, and a serialized computation itself. Full column information is stored
+// in the `columns` table, this one stores only entries for computed columns, so it will be empty for tables
+// without any computed columns defined in the schema. `computation` is a serialized blob and its format
+// is defined in column_computation.hh and system_schema docs.
+//
+static schema_ptr computed_columns_schema(const char* columns_table_name) {
+    schema_builder builder(make_lw_shared(::schema(generate_legacy_id(NAME, columns_table_name), NAME, columns_table_name,
+        // partition key
+        {{"keyspace_name", utf8_type}},
+        // clustering key
+        {{"table_name", utf8_type}, {"column_name", utf8_type}},
+        // regular columns
+        {{"computation", bytes_type}},
+        // static columns
+        {},
+        // regular column name type
+        utf8_type,
+        // comment
+        "computed columns"
+        )));
+    builder.set_gc_grace_seconds(schema_gc_grace);
+    builder.with_version(generate_schema_version(builder.uuid()));
+    return builder.build();
+}
+
+schema_ptr computed_columns() {
+    static thread_local auto schema = computed_columns_schema(COMPUTED_COLUMNS);
+    return schema;
+}
+
 schema_ptr dropped_columns() {
     static thread_local auto schema = [] {
         schema_builder builder(make_lw_shared(::schema(generate_legacy_id(NAME, DROPPED_COLUMNS), NAME, DROPPED_COLUMNS,
@@ -1662,6 +1701,7 @@ static schema_mutations make_table_mutations(schema_ptr table, api::timestamp_ty
     add_table_params_to_mutations(m, ckey, table, timestamp);
 
     mutation columns_mutation(columns(), pkey);
+    mutation computed_columns_mutation(computed_columns(), pkey);
     mutation dropped_columns_mutation(dropped_columns(), pkey);
     mutation indices_mutation(indexes(), pkey);
 
@@ -1671,6 +1711,9 @@ static schema_mutations make_table_mutations(schema_ptr table, api::timestamp_ty
                 throw std::logic_error("view_virtual column found in non-view table");
             }
             add_column_to_schema_mutation(table, column, timestamp, columns_mutation);
+            if (column.is_computed()) {
+                add_computed_column_to_schema_mutation(table, column, timestamp, computed_columns_mutation);
+            }
         }
         for (auto&& index : table->indices()) {
             add_index_to_schema_mutation(table, index, timestamp, indices_mutation);
@@ -1682,7 +1725,7 @@ static schema_mutations make_table_mutations(schema_ptr table, api::timestamp_ty
         }
     }
 
-    return schema_mutations{std::move(m), std::move(columns_mutation), std::nullopt,
+    return schema_mutations{std::move(m), std::move(columns_mutation), std::nullopt, std::move(computed_columns_mutation),
                             std::move(indices_mutation), std::move(dropped_columns_mutation),
                             std::move(scylla_tables_mutation)};
 }
@@ -1743,6 +1786,7 @@ static void make_update_columns_mutations(schema_ptr old_table,
         std::vector<mutation>& mutations) {
     mutation columns_mutation(columns(), partition_key::from_singular(*columns(), old_table->ks_name()));
     mutation view_virtual_columns_mutation(view_virtual_columns(), partition_key::from_singular(*columns(), old_table->ks_name()));
+    mutation computed_columns_mutation(computed_columns(), partition_key::from_singular(*columns(), old_table->ks_name()));
 
     auto diff = difference(old_table->v3().columns_by_name(), new_table->v3().columns_by_name());
 
@@ -1759,6 +1803,9 @@ static void make_update_columns_mutations(schema_ptr old_table,
         } else {
             drop_column_from_schema_mutation(columns(), old_table, column.name_as_text(), timestamp, mutations);
         }
+        if (column.is_computed()) {
+            drop_column_from_schema_mutation(computed_columns(), old_table, column.name_as_text(), timestamp, mutations);
+        }
     }
 
     // newly added columns and old columns with updated attributes
@@ -1769,10 +1816,14 @@ static void make_update_columns_mutations(schema_ptr old_table,
         } else {
             add_column_to_schema_mutation(new_table, column, timestamp, columns_mutation);
         }
+        if (column.is_computed()) {
+            add_computed_column_to_schema_mutation(new_table, column, timestamp, computed_columns_mutation);
+        }
     }
 
     mutations.emplace_back(std::move(columns_mutation));
     mutations.emplace_back(std::move(view_virtual_columns_mutation));
+    mutations.emplace_back(std::move(computed_columns_mutation));
 
     // dropped columns
     auto dc_diff = difference(old_table->dropped_columns(), new_table->dropped_columns());
@@ -1826,6 +1877,9 @@ static void make_drop_table_or_view_mutations(schema_ptr schema_table,
         } else {
             drop_column_from_schema_mutation(columns(), table_or_view, column.name_as_text(), timestamp, mutations);
         }
+        if (column.is_computed()) {
+            drop_column_from_schema_mutation(computed_columns(), table_or_view, column.name_as_text(), timestamp, mutations);
+        }
     }
     for (auto& column : table_or_view->dropped_columns() | boost::adaptors::map_keys) {
         drop_column_from_schema_mutation(dropped_columns(), table_or_view, column, timestamp, mutations);
@@ -1860,11 +1914,12 @@ static future<schema_mutations> read_table_mutations(distributed<service::storag
         read_schema_partition_for_table(proxy, s, table.keyspace_name, table.table_name),
         read_schema_partition_for_table(proxy, columns(), table.keyspace_name, table.table_name),
         read_schema_partition_for_table(proxy, view_virtual_columns(), table.keyspace_name, table.table_name),
+        read_schema_partition_for_table(proxy, computed_columns(), table.keyspace_name, table.table_name),
         read_schema_partition_for_table(proxy, dropped_columns(), table.keyspace_name, table.table_name),
         read_schema_partition_for_table(proxy, indexes(), table.keyspace_name, table.table_name),
         read_schema_partition_for_table(proxy, scylla_tables(), table.keyspace_name, table.table_name)).then(
-            [] (mutation cf_m, mutation col_m, mutation vv_col_m, mutation dropped_m, mutation idx_m, mutation st_m) {
-                return schema_mutations{std::move(cf_m), std::move(col_m), std::move(vv_col_m), std::move(idx_m), std::move(dropped_m), std::move(st_m)};
+            [] (mutation cf_m, mutation col_m, mutation vv_col_m, mutation c_col_m, mutation dropped_m, mutation idx_m, mutation st_m) {
+                return schema_mutations{std::move(cf_m), std::move(col_m), std::move(vv_col_m), std::move(c_col_m), std::move(idx_m), std::move(dropped_m), std::move(st_m)};
             });
 #if 0
         // FIXME:
@@ -2086,13 +2141,15 @@ schema_ptr create_table_from_mutations(const schema_ctxt& ctxt, schema_mutations
         }
     }
 
+    auto computed_columns = get_computed_columns(sm);
     std::vector<column_definition> column_defs = create_columns_from_column_rows(
             query::result_set(sm.columns_mutation()),
             ks_name,
             cf_name,/*,
             fullRawComparator, */
             cf == cf_type::super,
-            column_view_virtual::no);
+            column_view_virtual::no,
+            computed_columns);
 
 
     builder.set_is_dense(is_dense);
@@ -2165,6 +2222,16 @@ static void add_column_to_schema_mutation(schema_ptr table,
     m.set_clustered_cell(ckey, "type", type->as_cql3_type().to_string(), timestamp);
 }
 
+static void add_computed_column_to_schema_mutation(schema_ptr table,
+        const column_definition& column,
+        api::timestamp_type timestamp,
+        mutation& m) {
+    auto ckey = clustering_key::from_exploded(*m.schema(),
+            {utf8_type->decompose(table->cf_name()), utf8_type->decompose(column.name_as_text())});
+
+    m.set_clustered_cell(ckey, "computation", data_value(column.get_computation().serialize()), timestamp);
+}
+
 sstring serialize_kind(column_kind kind)
 {
     switch (kind) {
@@ -2250,12 +2317,24 @@ static void drop_column_from_schema_mutation(
     mutations.emplace_back(m);
 }
 
+static computed_columns_map get_computed_columns(const schema_mutations& sm) {
+    if (!sm.computed_columns_mutation()) {
+        return {};
+    }
+    query::result_set computed_result(*sm.computed_columns_mutation());
+    return boost::copy_range<computed_columns_map>(
+            computed_result.rows() | boost::adaptors::transformed([] (const query::result_set_row& row) {
+        return computed_columns_map::value_type{to_bytes(row.get_nonnull<sstring>("column_name")), column_computation::deserialize(row.get_nonnull<bytes>("computation"))};
+    }));
+}
+
 static std::vector<column_definition> create_columns_from_column_rows(const query::result_set& rows,
                                                                const sstring& keyspace,
                                                                const sstring& table, /*,
                                                                AbstractType<?> rawComparator, */
                                                                bool is_super,
-                                                               column_view_virtual is_view_virtual)
+                                                               column_view_virtual is_view_virtual,
+                                                               const computed_columns_map& computed_columns)
 {
     std::vector<column_definition> columns;
     for (auto&& row : rows.rows()) {
@@ -2271,8 +2350,13 @@ static std::vector<column_definition> create_columns_from_column_rows(const quer
                 type = reversed_type_impl::get_instance(type);
             }
         }
+        column_computation_ptr computation;
+        auto computed_it = computed_columns.find(name_bytes);
+        if (computed_it != computed_columns.end()) {
+            computation = computed_it->second->clone();
+        }
 
-        columns.emplace_back(name_bytes, type, kind, position, is_view_virtual);
+        columns.emplace_back(name_bytes, type, kind, position, is_view_virtual, std::move(computation));
     }
     return columns;
 }
@@ -2317,12 +2401,13 @@ view_ptr create_view_from_mutations(const schema_ctxt& ctxt, schema_mutations sm
     schema_builder builder{ks_name, cf_name, id};
     prepare_builder_from_table_row(ctxt, builder, row);
 
-    auto column_defs = create_columns_from_column_rows(query::result_set(sm.columns_mutation()), ks_name, cf_name, false, column_view_virtual::no);
+    auto computed_columns = get_computed_columns(sm);
+    auto column_defs = create_columns_from_column_rows(query::result_set(sm.columns_mutation()), ks_name, cf_name, false, column_view_virtual::no, computed_columns);
     for (auto&& cdef : column_defs) {
         builder.with_column(cdef);
     }
     if (sm.view_virtual_columns_mutation()) {
-        column_defs = create_columns_from_column_rows(query::result_set(*sm.view_virtual_columns_mutation()), ks_name, cf_name, false, column_view_virtual::yes);
+        column_defs = create_columns_from_column_rows(query::result_set(*sm.view_virtual_columns_mutation()), ks_name, cf_name, false, column_view_virtual::yes, computed_columns);
         for (auto&& cdef : column_defs) {
             builder.with_column(cdef);
         }
@@ -2394,9 +2479,9 @@ static schema_mutations make_view_mutations(view_ptr view, api::timestamp_type t
 
     add_table_params_to_mutations(m, ckey, view, timestamp);
 
-
     mutation columns_mutation(columns(), pkey);
     mutation view_virtual_columns_mutation(view_virtual_columns(), pkey);
+    mutation computed_columns_mutation(computed_columns(), pkey);
     mutation dropped_columns_mutation(dropped_columns(), pkey);
     mutation indices_mutation(indexes(), pkey);
 
@@ -2407,6 +2492,9 @@ static schema_mutations make_view_mutations(view_ptr view, api::timestamp_type t
             } else {
                 add_column_to_schema_mutation(view, column, timestamp, columns_mutation);
             }
+            if (column.is_computed()) {
+                add_computed_column_to_schema_mutation(view, column, timestamp, computed_columns_mutation);
+            }
         }
 
         for (auto&& e : view->dropped_columns()) {
@@ -2419,7 +2507,7 @@ static schema_mutations make_view_mutations(view_ptr view, api::timestamp_type t
 
     auto scylla_tables_mutation = make_scylla_tables_mutation(view, timestamp);
 
-    return schema_mutations{std::move(m), std::move(columns_mutation), std::move(view_virtual_columns_mutation),
+    return schema_mutations{std::move(m), std::move(columns_mutation), std::move(view_virtual_columns_mutation), std::move(computed_columns_mutation),
                             std::move(indices_mutation), std::move(dropped_columns_mutation),
                             std::move(scylla_tables_mutation)};
 }
@@ -2722,6 +2810,9 @@ std::vector<schema_ptr> all_tables(schema_features features) {
     if (features.contains<schema_feature::VIEW_VIRTUAL_COLUMNS>()) {
         result.emplace_back(view_virtual_columns());
     }
+    if (features.contains<schema_feature::COMPUTED_COLUMNS>()) {
+        result.emplace_back(computed_columns());
+    }
     return result;
 }
 
@@ -2730,6 +2821,39 @@ std::vector<sstring> all_table_names(schema_features features) {
            boost::adaptors::transformed([] (auto schema) { return schema->cf_name(); }));
 }
 
+future<> maybe_update_legacy_secondary_index_mv_schema(database& db, view_ptr v) {
+    // TODO(sarna): Remove once computed columns are guaranteed to be featured in the whole cluster.
+    // Legacy format for a secondary index used a hardcoded "token" column, which ensured a proper
+    // order for indexed queries. This "token" column is now implemented as a computed column,
+    // but for the sake of compatibility we assume that there might be indexes created in the legacy
+    // format, where "token" is not marked as computed. Once we're sure that all indexes have their
+    // columns marked as computed (because they were either created on a node that supports computed
+    // columns or were fixed by this utility function), it's safe to remove this function altogether.
+    if (!service::get_local_storage_service().cluster_supports_computed_columns()) {
+        return make_ready_future<>();
+    }
+
+    if (v->clustering_key_size() == 0) {
+        return make_ready_future<>();
+    }
+    const column_definition& first_view_ck = v->clustering_key_columns().front();
+    if (first_view_ck.is_computed()) {
+        return make_ready_future<>();
+    }
+
+    table& base = db.find_column_family(v->view_info()->base_id());
+    schema_ptr base_schema = base.schema();
+    // If the first clustering key part of a view is a column with name not found in base schema,
+    // it implies it might be backing an index created before computed columns were introduced,
+    // and as such it must be recreated properly.
+    if (base_schema->columns_by_name().count(first_view_ck.name()) == 0) {
+        schema_builder builder{schema_ptr(v)};
+        builder.mark_column_computed(first_view_ck.name(), std::make_unique<token_column_computation>());
+        return service::get_local_migration_manager().announce_view_update(view_ptr(builder.build()), true);
+    }
+    return make_ready_future<>();
+}
+
 namespace legacy {
 
 table_schema_version schema_mutations::digest() const {

db/schema_tables.hh

@@ -101,6 +101,7 @@ static constexpr auto FUNCTIONS = "functions";
 static constexpr auto AGGREGATES = "aggregates";
 static constexpr auto INDEXES = "indexes";
 static constexpr auto VIEW_VIRTUAL_COLUMNS = "view_virtual_columns"; // Scylla specific
+static constexpr auto COMPUTED_COLUMNS = "computed_columns"; // Scylla specific
 
 schema_ptr columns();
 schema_ptr view_virtual_columns();
@@ -109,6 +110,7 @@ schema_ptr indexes();
 schema_ptr tables();
 schema_ptr scylla_tables();
 schema_ptr views();
+schema_ptr computed_columns();
 
 }
 
@@ -206,6 +208,8 @@ std::vector<mutation> make_update_view_mutations(lw_shared_ptr<keyspace_metadata
 
 std::vector<mutation> make_drop_view_mutations(lw_shared_ptr<keyspace_metadata> keyspace, view_ptr view, api::timestamp_type timestamp);
 
+future<> maybe_update_legacy_secondary_index_mv_schema(database& db, view_ptr v);
+
 sstring serialize_kind(column_kind kind);
 column_kind deserialize_kind(sstring kind);
 data_type parse_type(sstring str);

db/view/view.cc

@@ -83,11 +83,18 @@ view_info::view_info(const schema& schema, const raw_view_info& raw_view_info)
 cql3::statements::select_statement& view_info::select_statement() const {
     if (!_select_statement) {
         shared_ptr<cql3::statements::raw::select_statement> raw;
-        if (is_index()) {
-            // Token column is the first clustering column
-            auto token_column_it = boost::range::find_if(_schema.all_columns(), std::mem_fn(&column_definition::is_clustering_key));
-            auto real_columns = _schema.all_columns() | boost::adaptors::filtered([this, token_column_it](const column_definition& cdef) {
-                return std::addressof(cdef) != std::addressof(*token_column_it);
+        // FIXME(sarna): legacy code, should be removed after "computed_columns" feature is guaranteed
+        // to be available on every node. Then, we won't need to check if this view is backing a secondary index.
+        const column_definition* legacy_token_column = nullptr;
+        if (service::get_local_storage_service().db().local().find_column_family(base_id()).get_index_manager().is_index(_schema)) {
+           if (!_schema.clustering_key_columns().empty()) {
+               legacy_token_column = &_schema.clustering_key_columns().front();
+           }
+        }
+
+        if (legacy_token_column || boost::algorithm::any_of(_schema.all_columns(), std::mem_fn(&column_definition::is_computed))) {
+            auto real_columns = _schema.all_columns() | boost::adaptors::filtered([this, legacy_token_column] (const column_definition& cdef) {
+                return &cdef != legacy_token_column && !cdef.is_computed();
             });
             schema::columns_type columns = boost::copy_range<schema::columns_type>(std::move(real_columns));
             raw = cql3::util::build_select_statement(base_name(), where_clause(), include_all_columns(), columns);
@@ -142,12 +149,6 @@ void view_info::initialize_base_dependent_fields(const schema& base) {
     }
 }
 
-bool view_info::is_index() const {
-    //TODO(sarna): result of this call can be cached instead of calling index_manager::is_index every time
-    column_family& base_cf = service::get_local_storage_service().db().local().find_column_family(base_id());
-    return base_cf.get_index_manager().is_index(view_ptr(_schema.shared_from_this()));
-}
-
 namespace db {
 
 namespace view {
@@ -248,7 +249,6 @@ private:
         return _updates.emplace(std::move(key), mutation_partition(_view)).first->second;
     }
     row_marker compute_row_marker(const clustering_row& base_row) const;
-    dht::token token_for(const partition_key& base_key);
     deletable_row& get_view_row(const partition_key& base_key, const clustering_row& update);
     bool can_skip_view_updates(const clustering_row& update, const clustering_row& existing) const;
     void create_entry(const partition_key& base_key, const clustering_row& update, gc_clock::time_point now);
@@ -297,20 +297,26 @@ row_marker view_updates::compute_row_marker(const clustering_row& base_row) cons
     return marker;
 }
 
-dht::token view_updates::token_for(const partition_key& base_key) {
-    return dht::global_partitioner().get_token(*_base, base_key);
-}
-
 deletable_row& view_updates::get_view_row(const partition_key& base_key, const clustering_row& update) {
     std::vector<bytes> linearized_values;
     auto get_value = boost::adaptors::transformed([&, this] (const column_definition& cdef) -> bytes_view {
         auto* base_col = _base->get_column_definition(cdef.name());
         if (!base_col) {
-            if (!_view_info.is_index()) {
+            bytes_opt computed_value;
+            if (!cdef.is_computed()) {
+                //FIXME(sarna): this legacy code is here for backward compatibility and should be removed
+                // once "computed_columns feature" is supported by every node
+                if (!service::get_local_storage_service().db().local().find_column_family(_base->id()).get_index_manager().is_index(*_base)) {
                     throw std::logic_error(format("Column {} doesn't exist in base and this view is not backing a secondary index", cdef.name_as_text()));
                 }
-            auto& partitioner = dht::global_partitioner();
-            return linearized_values.emplace_back(partitioner.token_to_bytes(token_for(base_key)));
+                computed_value = token_column_computation().compute_value(*_base, base_key, update);
+            } else {
+                computed_value = cdef.get_computation().compute_value(*_base, base_key, update);
+            }
+            if (!computed_value) {
+                throw std::logic_error(format("No value computed for primary key column {}", cdef.name()));
+            }
+            return linearized_values.emplace_back(*computed_value);
         }
         switch (base_col->kind) {
         case column_kind::partition_key:

docs/system_schema_keyspace.md

@@ -0,0 +1,40 @@
+# System schema keyspace layout
+
+This section describes layouts and usage of system\_schema.* tables.
+
+## system\_schema.computed\_columns
+
+Computed columns are a special kind of columns. Rather than having their value provided directly
+by the user, they are computed - possibly from other column values. Examples of such computed
+columns could be:
+ * token column generated from the base partition key for secondary indexes
+ * map value column, generated as the extraction of a single value from a map stored in a different column
+
+Computed columns in many ways act as regular columns, so they are also present in the `system_schema.columns` table -
+`system_schema.computed_columns` is an additional mapping that marks the column as computed and provides its computation.
+
+Schema:
+~~~
+CREATE TABLE system_schema.computed_columns (
+    keyspace_name text,
+    table_name text,
+    column_name text,
+    computation blob,
+    PRIMARY KEY (keyspace_name, table_name, column_name)
+) WITH CLUSTERING ORDER BY (table_name ASC, column_name ASC);
+~~~
+
+`computation` is stored as a blob and its contents are assumed to be a JSON representation of computation's type
+and any custom fields needed.
+
+Example representations:
+~~~
+{'type':'token'}
+
+{'type':'map_value','map':'my_map_column_name','key':'AF$^GESHHgge6yhf'}
+~~~
+
+The token computation does not need additional arguments, as it returns the token of base's partition key.
+In order to compute a map value, what's additionally needed is the column name that stores the map and the key
+at which the value is expected.
+

idl/frozen_schema.idl.hh

@@ -31,6 +31,7 @@ class schema_mutations {
     std::optional<canonical_mutation> dropped_columns_canonical_mutation()[[version 2.0]];
     std::optional<canonical_mutation> scylla_tables_canonical_mutation()[[version 2.0]];
     std::optional<canonical_mutation> view_virtual_columns_canonical_mutation()[[version 2.4]];
+    std::optional<canonical_mutation> computed_columns_canonical_mutation()[[version 3.2]];
 };
 
 class schema stub [[writable]] {

index/secondary_index_manager.cc

@@ -135,7 +135,7 @@ view_ptr secondary_index_manager::create_view_for_index(const index_metadata& im
         builder.with_column(index_target->name(), index_target->type, column_kind::partition_key);
         // Additional token column is added to ensure token order on secondary index queries
         bytes token_column_name = get_available_token_column_name(*schema);
-        builder.with_column(token_column_name, bytes_type, column_kind::clustering_key);
+        builder.with_computed_column(token_column_name, bytes_type, column_kind::clustering_key, std::make_unique<token_column_computation>());
         for (auto& col : schema->partition_key_columns()) {
             if (col == *index_target) {
                 continue;

schema.cc

@@ -497,12 +497,13 @@ sstring index_metadata::get_default_index_name(const sstring& cf_name,
     return cf_name + "_idx";
 }
 
-column_definition::column_definition(bytes name, data_type type, column_kind kind, column_id component_index, column_view_virtual is_view_virtual, api::timestamp_type dropped_at)
+column_definition::column_definition(bytes name, data_type type, column_kind kind, column_id component_index, column_view_virtual is_view_virtual, column_computation_ptr computation, api::timestamp_type dropped_at)
         : _name(std::move(name))
         , _dropped_at(dropped_at)
         , _is_atomic(type->is_atomic())
         , _is_counter(type->is_counter())
         , _is_view_virtual(is_view_virtual)
+        , _computation(std::move(computation))
         , type(std::move(type))
         , id(component_index)
         , kind(kind)
@@ -516,6 +517,9 @@ std::ostream& operator<<(std::ostream& os, const column_definition& cd) {
     if (cd.is_view_virtual()) {
         os << ", view_virtual";
     }
+    if (cd.is_computed()) {
+        os << ", computed:" << cd.get_computation().serialize();
+    }
     os << ", componentIndex=" << (cd.has_component_index() ? std::to_string(cd.component_index()) : "null");
     os << ", droppedAt=" << cd._dropped_at;
     os << "}";
@@ -701,7 +705,7 @@ column_definition& schema_builder::find_column(const cql3::column_identifier& c)
 }
 
 schema_builder& schema_builder::with_column(const column_definition& c) {
-    return with_column(bytes(c.name()), data_type(c.type), column_kind(c.kind), c.position(), c.view_virtual());
+    return with_column(bytes(c.name()), data_type(c.type), column_kind(c.kind), c.position(), c.view_virtual(), c.get_computation_ptr());
 }
 
 schema_builder& schema_builder::with_column(bytes name, data_type type, column_kind kind, column_view_virtual is_view_virtual) {
@@ -709,8 +713,8 @@ schema_builder& schema_builder::with_column(bytes name, data_type type, column_k
     return with_column(name, type, kind, 0, is_view_virtual);
 }
 
-schema_builder& schema_builder::with_column(bytes name, data_type type, column_kind kind, column_id component_index, column_view_virtual is_view_virtual) {
-    _raw._columns.emplace_back(name, type, kind, component_index, is_view_virtual);
+schema_builder& schema_builder::with_column(bytes name, data_type type, column_kind kind, column_id component_index, column_view_virtual is_view_virtual, column_computation_ptr computation) {
+    _raw._columns.emplace_back(name, type, kind, component_index, is_view_virtual, std::move(computation));
     if (type->is_multi_cell()) {
         with_collection(name, type);
     } else if (type->is_counter()) {
@@ -719,12 +723,18 @@ schema_builder& schema_builder::with_column(bytes name, data_type type, column_k
     return *this;
 }
 
+schema_builder& schema_builder::with_computed_column(bytes name, data_type type, column_kind kind, column_computation_ptr computation) {
+    return with_column(name, type, kind, 0, column_view_virtual::no, std::move(computation));
+}
+
 schema_builder& schema_builder::remove_column(bytes name)
 {
     auto it = boost::range::find_if(_raw._columns, [&] (auto& column) {
         return column.name() == name;
     });
-    assert(it != _raw._columns.end());
+    if(it == _raw._columns.end()) {
+        throw std::out_of_range(format("Cannot remove: column {} not found.", name));
+    }
     without_column(it->name_as_text(), it->type, api::new_timestamp());
     _raw._columns.erase(it);
     return *this;
@@ -770,6 +780,14 @@ schema_builder& schema_builder::alter_column_type(bytes name, data_type new_type
     return *this;
 }
 
+schema_builder& schema_builder::mark_column_computed(bytes name, column_computation_ptr computation) {
+    auto it = boost::find_if(_raw._columns, [&name] (const column_definition& c) { return c.name() == name; });
+    assert(it != _raw._columns.end());
+    it->set_computed(std::move(computation));
+
+    return *this;
+}
+
 schema_builder& schema_builder::with_collection(bytes name, data_type type)
 {
     _raw._collections.emplace(name, type);
@@ -1244,6 +1262,36 @@ raw_view_info::raw_view_info(utils::UUID base_id, sstring base_name, bool includ
         , _where_clause(where_clause)
 { }
 
+column_computation_ptr column_computation::deserialize(bytes_view raw) {
+    return deserialize(json::to_json_value(sstring(raw.begin(), raw.end())));
+}
+
+column_computation_ptr column_computation::deserialize(const Json::Value& parsed) {
+    if (!parsed.isObject()) {
+        throw std::runtime_error(format("Invalid column computation value: {}", parsed.toStyledString()));
+    }
+    Json::Value type_json = parsed.get("type", Json::Value());
+    if (!type_json.isString()) {
+        throw std::runtime_error(format("Type {} is not convertible to string", type_json.toStyledString()));
+    }
+    sstring type = type_json.asString();
+    if (type == "token") {
+        return std::make_unique<token_column_computation>();
+    }
+    throw std::runtime_error(format("Incorrect column computation type {} found when parsing {}", type, parsed.toStyledString()));
+}
+
+bytes token_column_computation::serialize() const {
+    Json::Value serialized(Json::objectValue);
+    serialized["type"] = Json::Value("token");
+    return to_bytes(json::to_sstring(serialized));
+}
+
+bytes_opt token_column_computation::compute_value(const schema& schema, const partition_key& key, const clustering_row& row) const {
+    dht::i_partitioner& partitioner = dht::global_partitioner();
+    return partitioner.token_to_bytes(partitioner.get_token(schema, key));
+}
+
 bool operator==(const raw_view_info& x, const raw_view_info& y) {
     return x._base_id == y._base_id
         && x._base_name == y._base_name

schema.hh

@@ -39,6 +39,7 @@
 #include "compress.hh"
 #include "compaction_strategy.hh"
 #include "caching_options.hh"
+#include "column_computation.hh"
 
 using column_count_type = uint32_t;
 
@@ -217,6 +218,7 @@ private:
     bool _is_atomic;
     bool _is_counter;
     column_view_virtual _is_view_virtual;
+    column_computation_ptr _computation;
 
     struct thrift_bits {
         thrift_bits()
@@ -232,6 +234,7 @@ public:
     column_definition(bytes name, data_type type, column_kind kind,
         column_id component_index = 0,
         column_view_virtual view_virtual = column_view_virtual::no,
+        column_computation_ptr = nullptr,
         api::timestamp_type dropped_at = api::missing_timestamp);
 
     data_type type;
@@ -244,6 +247,35 @@ public:
     column_kind kind;
     ::shared_ptr<cql3::column_specification> column_specification;
 
+    // NOTICE(sarna): This copy constructor is hand-written instead of default,
+    // because it involves deep copying of the computation object.
+    // Computation has a strict ownership policy provided by
+    // unique_ptr, and as such cannot rely on default copying.
+    column_definition(const column_definition& other)
+            : _name(other._name)
+            , _dropped_at(other._dropped_at)
+            , _is_atomic(other._is_atomic)
+            , _is_counter(other._is_counter)
+            , _is_view_virtual(other._is_view_virtual)
+            , _computation(other.get_computation_ptr())
+            , _thrift_bits(other._thrift_bits)
+            , type(other.type)
+            , id(other.id)
+            , kind(other.kind)
+            , column_specification(other.column_specification)
+        {}
+
+    column_definition& operator=(const column_definition& other) {
+        if (this == &other) {
+            return *this;
+        }
+        column_definition tmp(other);
+        *this = std::move(tmp);
+        return *this;
+    }
+
+    column_definition& operator=(column_definition&& other) = default;
+
     bool is_static() const { return kind == column_kind::static_column; }
     bool is_regular() const { return kind == column_kind::regular_column; }
     bool is_partition_key() const { return kind == column_kind::partition_key; }
@@ -258,6 +290,14 @@ public:
     // These columns should be hidden from the user's SELECT queries.
     bool is_view_virtual() const { return _is_view_virtual == column_view_virtual::yes; }
     column_view_virtual view_virtual() const { return _is_view_virtual; }
+    // Computed column values are generated from other columns (and possibly other sources) during updates.
+    // Their values are still stored on disk, same as a regular columns.
+    bool is_computed() const { return bool(_computation); }
+    const column_computation& get_computation() const { return *_computation; }
+    column_computation_ptr get_computation_ptr() const {
+        return _computation ? _computation->clone() : nullptr;
+    }
+    void set_computed(column_computation_ptr computation) { _computation = std::move(computation); }
     // Columns hidden from CQL cannot be in any way retrieved by the user,
     // either explicitly or via the '*' operator, or functions, aggregates, etc.
     bool is_hidden_from_cql() const { return is_view_virtual(); }

schema_builder.hh

@@ -239,12 +239,14 @@ public:
     column_definition& find_column(const cql3::column_identifier&);
     schema_builder& with_column(const column_definition& c);
     schema_builder& with_column(bytes name, data_type type, column_kind kind = column_kind::regular_column, column_view_virtual view_virtual = column_view_virtual::no);
-    schema_builder& with_column(bytes name, data_type type, column_kind kind, column_id component_index, column_view_virtual view_virtual = column_view_virtual::no);
+    schema_builder& with_column(bytes name, data_type type, column_kind kind, column_id component_index, column_view_virtual view_virtual = column_view_virtual::no, column_computation_ptr computation = nullptr);
+    schema_builder& with_computed_column(bytes name, data_type type, column_kind kind, column_computation_ptr computation);
     schema_builder& remove_column(bytes name);
     schema_builder& without_column(sstring name, api::timestamp_type timestamp);
     schema_builder& without_column(sstring name, data_type, api::timestamp_type timestamp);
     schema_builder& rename_column(bytes from, bytes to);
     schema_builder& alter_column_type(bytes name, data_type new_type);
+    schema_builder& mark_column_computed(bytes name, column_computation_ptr computation);
 
     // Adds information about collection that existed in the past but the column
     // has since been removed. For adding colllections that are still alive

schema_mutations.cc

@@ -30,10 +30,12 @@ schema_mutations::schema_mutations(canonical_mutation columnfamilies,
                                    std::optional<canonical_mutation> indices,
                                    std::optional<canonical_mutation> dropped_columns,
                                    std::optional<canonical_mutation> scylla_tables,
-                                   std::optional<canonical_mutation> view_virtual_columns)
+                                   std::optional<canonical_mutation> view_virtual_columns,
+                                   std::optional<canonical_mutation> computed_columns)
     : _columnfamilies(columnfamilies.to_mutation(is_view ? db::schema_tables::views() : db::schema_tables::tables()))
     , _columns(columns.to_mutation(db::schema_tables::columns()))
     , _view_virtual_columns(view_virtual_columns ? mutation_opt{view_virtual_columns.value().to_mutation(db::schema_tables::view_virtual_columns())} : std::nullopt)
+    , _computed_columns(computed_columns ? mutation_opt{computed_columns.value().to_mutation(db::schema_tables::computed_columns())} : std::nullopt)
     , _indices(indices ? mutation_opt{indices.value().to_mutation(db::schema_tables::indexes())} : std::nullopt)
     , _dropped_columns(dropped_columns ? mutation_opt{dropped_columns.value().to_mutation(db::schema_tables::dropped_columns())} : std::nullopt)
     , _scylla_tables(scylla_tables ? mutation_opt{scylla_tables.value().to_mutation(db::schema_tables::scylla_tables())} : std::nullopt)
@@ -45,6 +47,9 @@ void schema_mutations::copy_to(std::vector<mutation>& dst) const {
     if (_view_virtual_columns) {
         dst.push_back(*_view_virtual_columns);
     }
+    if (_computed_columns) {
+        dst.push_back(*_computed_columns);
+    }
     if (_indices) {
         dst.push_back(*_indices);
     }
@@ -85,6 +90,9 @@ table_schema_version schema_mutations::digest() const {
     if (_view_virtual_columns && !_view_virtual_columns->partition().empty()) {
         db::schema_tables::feed_hash_for_schema_digest(h, *_view_virtual_columns, sf);
     }
+    if (_computed_columns && !_computed_columns->partition().empty()) {
+        db::schema_tables::feed_hash_for_schema_digest(h, *_computed_columns, sf);
+    }
     if (_indices && !_indices->partition().empty()) {
         db::schema_tables::feed_hash_for_schema_digest(h, *_indices, sf);
     }
@@ -112,6 +120,7 @@ bool schema_mutations::operator==(const schema_mutations& other) const {
     return compact(_columnfamilies) == compact(other._columnfamilies)
            && compact(_columns) == compact(other._columns)
            && compact(_view_virtual_columns) == compact(other._view_virtual_columns)
+           && compact(_computed_columns) == compact(other._computed_columns)
            && compact(_indices) == compact(other._indices)
            && compact(_dropped_columns) == compact(other._dropped_columns)
            && compact(_scylla_tables) == compact(other._scylla_tables)
@@ -124,7 +133,8 @@ bool schema_mutations::operator!=(const schema_mutations& other) const {
 
 bool schema_mutations::live() const {
     return _columnfamilies.live_row_count() > 0 || _columns.live_row_count() > 0 ||
-            (_view_virtual_columns && _view_virtual_columns->live_row_count() > 0);
+            (_view_virtual_columns && _view_virtual_columns->live_row_count() > 0) ||
+            (_computed_columns && _computed_columns->live_row_count() > 0);
 }
 
 bool schema_mutations::is_view() const {

schema_mutations.hh

@@ -32,15 +32,17 @@ class schema_mutations {
     mutation _columnfamilies;
     mutation _columns;
     mutation_opt _view_virtual_columns;
+    mutation_opt _computed_columns;
     mutation_opt _indices;
     mutation_opt _dropped_columns;
     mutation_opt _scylla_tables;
 public:
-    schema_mutations(mutation columnfamilies, mutation columns, mutation_opt view_virtual_columns, mutation_opt indices, mutation_opt dropped_columns,
+    schema_mutations(mutation columnfamilies, mutation columns, mutation_opt view_virtual_columns, mutation_opt computed_columns, mutation_opt indices, mutation_opt dropped_columns,
         mutation_opt scylla_tables)
             : _columnfamilies(std::move(columnfamilies))
             , _columns(std::move(columns))
             , _view_virtual_columns(std::move(view_virtual_columns))
+            , _computed_columns(std::move(computed_columns))
             , _indices(std::move(indices))
             , _dropped_columns(std::move(dropped_columns))
             , _scylla_tables(std::move(scylla_tables))
@@ -51,7 +53,8 @@ public:
                      std::optional<canonical_mutation> indices,
                      std::optional<canonical_mutation> dropped_columns,
                      std::optional<canonical_mutation> scylla_tables,
-                     std::optional<canonical_mutation> view_virtual_columns);
+                     std::optional<canonical_mutation> view_virtual_columns,
+                     std::optional<canonical_mutation> computed_columns);
 
     schema_mutations(schema_mutations&&) = default;
     schema_mutations& operator=(schema_mutations&&) = default;
@@ -72,6 +75,10 @@ public:
         return _view_virtual_columns;
     }
 
+    const mutation_opt& computed_columns_mutation() const {
+        return _computed_columns;
+    }
+
     const mutation_opt& scylla_tables() const {
         return _scylla_tables;
     }
@@ -102,6 +109,13 @@ public:
         return {};
     }
 
+    std::optional<canonical_mutation> computed_columns_canonical_mutation() const {
+        if (_computed_columns) {
+            return canonical_mutation(*_computed_columns);
+        }
+        return {};
+    }
+
     std::optional<canonical_mutation> indices_canonical_mutation() const {
         if (_indices) {
             return canonical_mutation(*_indices);

service/storage_service.cc

@@ -111,6 +111,7 @@ static const sstring CORRECT_STATIC_COMPACT_IN_MC = "CORRECT_STATIC_COMPACT_IN_M
 static const sstring UNBOUNDED_RANGE_TOMBSTONES_FEATURE = "UNBOUNDED_RANGE_TOMBSTONES";
 static const sstring VIEW_VIRTUAL_COLUMNS = "VIEW_VIRTUAL_COLUMNS";
 static const sstring DIGEST_INSENSITIVE_TO_EXPIRY = "DIGEST_INSENSITIVE_TO_EXPIRY";
+static const sstring COMPUTED_COLUMNS_FEATURE = "COMPUTED_COLUMNS";
 
 static const sstring SSTABLE_FORMAT_PARAM_NAME = "sstable_format";
 
@@ -164,6 +165,7 @@ storage_service::storage_service(distributed<database>& db, gms::gossiper& gossi
         , _unbounded_range_tombstones_feature(_feature_service, UNBOUNDED_RANGE_TOMBSTONES_FEATURE)
         , _view_virtual_columns(_feature_service, VIEW_VIRTUAL_COLUMNS)
         , _digest_insensitive_to_expiry(_feature_service, DIGEST_INSENSITIVE_TO_EXPIRY)
+        , _computed_columns(_feature_service, COMPUTED_COLUMNS_FEATURE)
         , _la_feature_listener(*this, _feature_listeners_sem, sstables::sstable_version_types::la)
         , _mc_feature_listener(*this, _feature_listeners_sem, sstables::sstable_version_types::mc)
         , _replicate_action([this] { return do_replicate_to_all_cores(); })
@@ -218,6 +220,7 @@ void storage_service::enable_all_features() {
         std::ref(_unbounded_range_tombstones_feature),
         std::ref(_view_virtual_columns),
         std::ref(_digest_insensitive_to_expiry),
+        std::ref(_computed_columns),
     })
     {
         if (features.count(f.name())) {
@@ -322,6 +325,7 @@ std::set<sstring> storage_service::get_config_supported_features_set() {
         CORRECT_STATIC_COMPACT_IN_MC,
         VIEW_VIRTUAL_COLUMNS,
         DIGEST_INSENSITIVE_TO_EXPIRY,
+        COMPUTED_COLUMNS_FEATURE,
     };
 
     // Do not respect config in the case database is not started
@@ -3508,6 +3512,7 @@ db::schema_features storage_service::cluster_schema_features() const {
     db::schema_features f;
     f.set_if<db::schema_feature::VIEW_VIRTUAL_COLUMNS>(bool(_view_virtual_columns));
     f.set_if<db::schema_feature::DIGEST_INSENSITIVE_TO_EXPIRY>(bool(_digest_insensitive_to_expiry));
+    f.set_if<db::schema_feature::COMPUTED_COLUMNS>(bool(_computed_columns));
     return f;
 }
 

service/storage_service.hh

@@ -332,6 +332,7 @@ private:
     gms::feature _unbounded_range_tombstones_feature;
     gms::feature _view_virtual_columns;
     gms::feature _digest_insensitive_to_expiry;
+    gms::feature _computed_columns;
 
     sstables::sstable_version_types _sstables_format = sstables::sstable_version_types::ka;
     seastar::semaphore _feature_listeners_sem = {1};
@@ -2344,14 +2345,21 @@ public:
     bool cluster_supports_unbounded_range_tombstones() const {
         return bool(_unbounded_range_tombstones_feature);
     }
+
     const gms::feature& cluster_supports_view_virtual_columns() const {
         return _view_virtual_columns;
     }
     const gms::feature& cluster_supports_digest_insensitive_to_expiry() const {
         return _digest_insensitive_to_expiry;
     }
+
+    bool cluster_supports_computed_columns() const {
+        return bool(_computed_columns);
+    }
+
     // Returns schema features which all nodes in the cluster advertise as supported.
     db::schema_features cluster_schema_features() const;
+
 private:
     future<> set_cql_ready(bool ready);
 private:

tests/schema_change_test.cc

@@ -510,15 +510,17 @@ SEASTAR_TEST_CASE(test_prepared_statement_is_invalidated_by_schema_change) {
 
 // We don't want schema digest to change between Scylla versions because that results in a schema disagreement
 // during rolling upgrade.
-SEASTAR_TEST_CASE(test_schema_digest_does_not_change) {
+future<> test_schema_digest_does_not_change_with_disabled_features(sstring data_dir, std::set<sstring> disabled_features, std::vector<utils::UUID> expected_digests) {
     using namespace db;
     using namespace db::schema_tables;
 
     auto tmp = tmpdir();
+    // NOTICE: Regenerating data for this test may be necessary when a system table is added.
+    // This test uses pre-generated sstables and relies on the fact that they are up to date
+    // with the current system schema. If it is not, the schema will be updated, which will cause
+    // new timestamps to appear and schema digests will not match anymore.
     const bool regenerate = false;
 
-    sstring data_dir = "./tests/sstables/schema_digest_test";
-
     auto db_cfg_ptr = make_shared<db::config>();
     auto& db_cfg = *db_cfg_ptr;
     if (regenerate) {
@@ -527,8 +529,10 @@ SEASTAR_TEST_CASE(test_schema_digest_does_not_change) {
         fs::copy(std::string(data_dir), std::string(tmp.path().string()), fs::copy_options::recursive);
         db_cfg.data_file_directories({tmp.path().string()}, db::config::config_source::CommandLine);
     }
+    cql_test_config cfg_in(db_cfg_ptr);
+    cfg_in.disabled_features = std::move(disabled_features);
 
-    return do_with_cql_env_thread([regenerate](cql_test_env& e) {
+    return do_with_cql_env_thread([regenerate, expected_digests = std::move(expected_digests)](cql_test_env& e) {
         if (regenerate) {
             // Exercise many different kinds of schema changes.
             e.execute_cql(
@@ -566,25 +570,58 @@ SEASTAR_TEST_CASE(test_schema_digest_does_not_change) {
 
         schema_features sf = schema_features::of<schema_feature::DIGEST_INSENSITIVE_TO_EXPIRY>();
 
-        expect_digest(sf, utils::UUID("492719e5-0169-30b1-a15e-3447674c0c0c"));
+        expect_digest(sf, expected_digests[0]);
 
         sf.set<schema_feature::VIEW_VIRTUAL_COLUMNS>();
-        expect_digest(sf, utils::UUID("be3c0af4-417f-31d5-8e0e-4ac257ec00ad"));
+        expect_digest(sf, expected_digests[1]);
 
-        expect_digest(schema_features::full(), utils::UUID("be3c0af4-417f-31d5-8e0e-4ac257ec00ad"));
+        sf.set<schema_feature::VIEW_VIRTUAL_COLUMNS>();
+        expect_digest(sf, expected_digests[2]);
+
+        expect_digest(schema_features::full(), expected_digests[3]);
 
         // Causes tombstones to become expired
         // This is in order to test that schema disagreement doesn't form due to expired tombstones being collected
         // Refs https://github.com/scylladb/scylla/issues/4485
         forward_jump_clocks(std::chrono::seconds(60*60*24*31));
 
-        expect_digest(schema_features::full(), utils::UUID("be3c0af4-417f-31d5-8e0e-4ac257ec00ad"));
+        expect_digest(schema_features::full(), expected_digests[4]);
 
         // FIXME: schema_mutations::digest() is still sensitive to expiry, so we can check versions only after forward_jump_clocks()
         // otherwise the results would not be stable.
-        expect_version("tests", "table1", utils::UUID("4198e26c-f214-3888-9c49-c396eb01b8d7"));
-        expect_version("ks", "tbl", utils::UUID("5c9cadec-e5df-357e-81d0-0261530af64b"));
-        expect_version("ks", "tbl_view", utils::UUID("1d91ad22-ea7c-3e7f-9557-87f0f3bb94d7"));
-        expect_version("ks", "tbl_view_2", utils::UUID("2dcd4a37-cbb5-399b-b3c9-8eb1398b096b"));
-    }, db_cfg_ptr).then([tmp = std::move(tmp)] {});
+        expect_version("tests", "table1", expected_digests[5]);
+        expect_version("ks", "tbl", expected_digests[6]);
+        expect_version("ks", "tbl_view", expected_digests[7]);
+        expect_version("ks", "tbl_view_2", expected_digests[8]);
+    }, cfg_in).then([tmp = std::move(tmp)] {});
+}
+
+SEASTAR_TEST_CASE(test_schema_digest_does_not_change) {
+    std::vector<utils::UUID> expected_digests{
+        utils::UUID("492719e5-0169-30b1-a15e-3447674c0c0c"),
+        utils::UUID("be3c0af4-417f-31d5-8e0e-4ac257ec00ad"),
+        utils::UUID("be3c0af4-417f-31d5-8e0e-4ac257ec00ad"),
+        utils::UUID("be3c0af4-417f-31d5-8e0e-4ac257ec00ad"),
+        utils::UUID("be3c0af4-417f-31d5-8e0e-4ac257ec00ad"),
+        utils::UUID("4198e26c-f214-3888-9c49-c396eb01b8d7"),
+        utils::UUID("5c9cadec-e5df-357e-81d0-0261530af64b"),
+        utils::UUID("1d91ad22-ea7c-3e7f-9557-87f0f3bb94d7"),
+        utils::UUID("2dcd4a37-cbb5-399b-b3c9-8eb1398b096b")
+    };
+    return test_schema_digest_does_not_change_with_disabled_features("./tests/sstables/schema_digest_test", std::set<sstring>{"COMPUTED_COLUMNS"}, std::move(expected_digests));
+}
+
+SEASTAR_TEST_CASE(test_schema_digest_does_not_change_after_computed_columns) {
+    std::vector<utils::UUID> expected_digests{
+        utils::UUID("ddd2b841-1bbb-374a-972c-037d6bc14d28"),
+        utils::UUID("ea8433b3-d150-3c93-8249-a584537c1b4e"),
+        utils::UUID("ea8433b3-d150-3c93-8249-a584537c1b4e"),
+        utils::UUID("9837e11f-13b8-32ba-9171-5563248dc198"),
+        utils::UUID("9837e11f-13b8-32ba-9171-5563248dc198"),
+        utils::UUID("774d63ef-2f75-39f8-a2be-418d28d35a97"),
+        utils::UUID("5217fc3a-308f-32aa-8b9c-41a6f2bcc448"),
+        utils::UUID("d58e5214-516e-3d0b-95b5-01ab71584a8d"),
+        utils::UUID("e1b50bed-2ab8-3759-92c7-1f4288046ae6")
+    };
+    return test_schema_digest_does_not_change_with_disabled_features("./tests/sstables/schema_digest_test_computed_columns", std::set<sstring>{}, std::move(expected_digests));
 }

tests/secondary_index_test.cc

@@ -1195,3 +1195,31 @@ SEASTAR_TEST_CASE(test_indexing_paging_and_aggregation) {
       });
     });
 }
+
+SEASTAR_TEST_CASE(test_computed_columns) {
+    return do_with_cql_env_thread([] (auto& e) {
+        e.execute_cql("CREATE TABLE t (p1 int, p2 int, c1 int, c2 int, v int, PRIMARY KEY ((p1,p2),c1,c2))").get();
+        e.execute_cql("CREATE INDEX local1 ON t ((p1,p2),v)").get();
+        e.execute_cql("CREATE INDEX global1 ON t (v)").get();
+        e.execute_cql("CREATE INDEX global2 ON t (c2)").get();
+        e.execute_cql("CREATE INDEX local2 ON t ((p1,p2),c2)").get();
+
+        auto local1 = e.local_db().find_schema("ks", "local1_index");
+        auto local2 = e.local_db().find_schema("ks", "local2_index");
+        auto global1 = e.local_db().find_schema("ks", "global1_index");
+        auto global2 = e.local_db().find_schema("ks", "global2_index");
+
+        bytes token_column_name("idx_token");
+        data_value token_computation(token_column_computation().serialize());
+        BOOST_REQUIRE_EQUAL(local1->get_column_definition(token_column_name), nullptr);
+        BOOST_REQUIRE_EQUAL(local2->get_column_definition(token_column_name), nullptr);
+        BOOST_REQUIRE(global1->get_column_definition(token_column_name)->is_computed());
+        BOOST_REQUIRE(global2->get_column_definition(token_column_name)->is_computed());
+
+        auto msg = e.execute_cql("SELECT computation FROM system_schema.computed_columns WHERE keyspace_name='ks'").get0();
+        assert_that(msg).is_rows().with_rows({
+            {{bytes_type->decompose(token_computation)}},
+            {{bytes_type->decompose(token_computation)}}
+        });
+    });
+}

tests/sstables/schema_digest_test_computed_columns/system/IndexInfo-9f5c6374d48532299a0a5094af9ad1e3/mc-1-big-Digest.crc32

@@ -0,0 +1 @@
+3401885169

tests/sstables/schema_digest_test_computed_columns/system/IndexInfo-9f5c6374d48532299a0a5094af9ad1e3/mc-1-big-TOC.txt

@@ -0,0 +1,9 @@
+Scylla.db
+CompressionInfo.db
+Filter.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Index.db
+Summary.db
+Data.db

tests/sstables/schema_digest_test_computed_columns/system/built_views-4b3c50a9ea873d7691016dbc9c38494a/mc-1-big-Digest.crc32

@@ -0,0 +1 @@
+2185033598

tests/sstables/schema_digest_test_computed_columns/system/built_views-4b3c50a9ea873d7691016dbc9c38494a/mc-1-big-TOC.txt

@@ -0,0 +1,9 @@
+Scylla.db
+CompressionInfo.db
+Filter.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Index.db
+Summary.db
+Data.db

tests/sstables/schema_digest_test_computed_columns/system/compaction_history-b4dbb7b4dc493fb5b3bfce6e434832ca/mc-1-big-Digest.crc32

@@ -0,0 +1 @@
+1020716506

tests/sstables/schema_digest_test_computed_columns/system/compaction_history-b4dbb7b4dc493fb5b3bfce6e434832ca/mc-1-big-TOC.txt

@@ -0,0 +1,9 @@
+Scylla.db
+CompressionInfo.db
+Filter.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Index.db
+Summary.db
+Data.db

tests/sstables/schema_digest_test_computed_columns/system/local-7ad54392bcdd35a684174e047860b377/mc-7-big-Digest.crc32

@@ -0,0 +1 @@
+259816102

tests/sstables/schema_digest_test_computed_columns/system/local-7ad54392bcdd35a684174e047860b377/mc-7-big-TOC.txt

@@ -0,0 +1,9 @@
+Scylla.db
+CompressionInfo.db
+Filter.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Index.db
+Summary.db
+Data.db

tests/sstables/schema_digest_test_computed_columns/system/local-7ad54392bcdd35a684174e047860b377/mc-8-big-Digest.crc32

@@ -0,0 +1 @@
+3439255993

tests/sstables/schema_digest_test_computed_columns/system/local-7ad54392bcdd35a684174e047860b377/mc-8-big-TOC.txt

@@ -0,0 +1,9 @@
+Scylla.db
+CompressionInfo.db
+Filter.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Index.db
+Summary.db
+Data.db

tests/sstables/schema_digest_test_computed_columns/system/peers-37f71aca7dc2383ba70672528af04d4f/mc-3-big-Digest.crc32

@@ -0,0 +1 @@
+3295636296

tests/sstables/schema_digest_test_computed_columns/system/peers-37f71aca7dc2383ba70672528af04d4f/mc-3-big-TOC.txt

@@ -0,0 +1,9 @@
+Scylla.db
+CompressionInfo.db
+Filter.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Index.db
+Summary.db
+Data.db

tests/sstables/schema_digest_test_computed_columns/system/scylla_views_builds_in_progress-a04c7bfd1e1336c9a44df22da352281d/mc-1-big-Digest.crc32

@@ -0,0 +1 @@
+3801415661

tests/sstables/schema_digest_test_computed_columns/system/scylla_views_builds_in_progress-a04c7bfd1e1336c9a44df22da352281d/mc-1-big-TOC.txt

@@ -0,0 +1,9 @@
+Scylla.db
+CompressionInfo.db
+Filter.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Index.db
+Summary.db
+Data.db

tests/sstables/schema_digest_test_computed_columns/system_auth/roles-5bc52802de2535edaeab188eecebb090/mc-1-big-Digest.crc32

@@ -0,0 +1 @@
+2300498069

tests/sstables/schema_digest_test_computed_columns/system_auth/roles-5bc52802de2535edaeab188eecebb090/mc-1-big-TOC.txt

@@ -0,0 +1,9 @@
+Scylla.db
+CompressionInfo.db
+Filter.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Index.db
+Summary.db
+Data.db

tests/sstables/schema_digest_test_computed_columns/system_distributed/view_build_status-5582b59f8e4e35e1b9133acada51eb04/mc-1-big-Digest.crc32

@@ -0,0 +1 @@
+1730504014

tests/sstables/schema_digest_test_computed_columns/system_distributed/view_build_status-5582b59f8e4e35e1b9133acada51eb04/mc-1-big-TOC.txt

@@ -0,0 +1,9 @@
+Scylla.db
+CompressionInfo.db
+Filter.db
+Statistics.db
+TOC.txt
+Digest.crc32
+Index.db
+Summary.db
+Data.db