From 07984215a34d057b2e47bbc3c4c1dc594224da80 Mon Sep 17 00:00:00 2001
From: Kamil Braun <kbraun@scylladb.com>
Date: Fri, 8 Sep 2023 19:51:29 +0200
Subject: [PATCH] feature_service: add `GROUP0_SCHEMA_VERSIONING` feature

This feature, when enabled, will modify how schema versions
are calculated and stored.

- In group 0 mode, schema versions are persisted by the group 0 command
  that performs the schema change, then reused by each node instead of
  being calculated as a digest (hash) by each node independently.
- In RECOVERY mode or before Raft upgrade procedure finishes, when we
  perform a schema change, we revert to the old digest-based way, taking
  into account the possibility of having performed group0-mode schema
  changes (that used persistent versions). As we will see in future
  commits, this will be done by storing additional flags and tombstones
  in system tables.

By "schema versions" we mean both the UUIDs returned from
`schema::version()` and the "global" schema version (the one we gossip
as `application_state::SCHEMA`).

For now, in this commit, the feature is always disabled. Once all
necessary code is setup in following commits, we will enable it together
with Raft.
---
 db/schema_features.hh  | 6 +++++-
 gms/feature_service.cc | 3 +++
 gms/feature_service.hh | 8 ++++++++
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/db/schema_features.hh b/db/schema_features.hh
index 2ca4fd768e..b178261b4c 100644
--- a/db/schema_features.hh
+++ b/db/schema_features.hh
@@ -28,6 +28,9 @@ enum class schema_feature {
     // When enabled, schema_mutations::digest() will skip empty mutations (with only tombstones),
     // so that the digest remains the same after schema tables are compacted.
     TABLE_DIGEST_INSENSITIVE_TO_EXPIRY,
+
+    // When enabled we'll add a new column to the `system_schema.scylla_tables` table.
+    GROUP0_SCHEMA_VERSIONING,
 };
 
 using schema_features = enum_set<super_enum<schema_feature,
@@ -38,7 +41,8 @@ using schema_features = enum_set<super_enum<schema_feature,
     schema_feature::PER_TABLE_PARTITIONERS,
     schema_feature::SCYLLA_KEYSPACES,
     schema_feature::SCYLLA_AGGREGATES,
-    schema_feature::TABLE_DIGEST_INSENSITIVE_TO_EXPIRY
+    schema_feature::TABLE_DIGEST_INSENSITIVE_TO_EXPIRY,
+    schema_feature::GROUP0_SCHEMA_VERSIONING
     >>;
 
 }
diff --git a/gms/feature_service.cc b/gms/feature_service.cc
index feb4d1fcf8..af51156725 100644
--- a/gms/feature_service.cc
+++ b/gms/feature_service.cc
@@ -76,6 +76,8 @@ feature_config feature_config_from_db_config(const db::config& cfg, std::set<sst
         }
     }
 
+    fcfg._disabled_features.insert("GROUP0_SCHEMA_VERSIONING"s);
+
     if (!cfg.check_experimental(db::experimental_features_t::feature::ALTERNATOR_STREAMS)) {
         fcfg._disabled_features.insert("ALTERNATOR_STREAMS"s);
     }
@@ -203,6 +205,7 @@ db::schema_features feature_service::cluster_schema_features() const {
     f.set_if<db::schema_feature::SCYLLA_KEYSPACES>(keyspace_storage_options);
     f.set_if<db::schema_feature::SCYLLA_AGGREGATES>(aggregate_storage_options);
     f.set_if<db::schema_feature::TABLE_DIGEST_INSENSITIVE_TO_EXPIRY>(table_digest_insensitive_to_expiry);
+    f.set_if<db::schema_feature::GROUP0_SCHEMA_VERSIONING>(group0_schema_versioning);
     return f;
 }
 
diff --git a/gms/feature_service.hh b/gms/feature_service.hh
index 03c4823d47..abc5e441bd 100644
--- a/gms/feature_service.hh
+++ b/gms/feature_service.hh
@@ -129,6 +129,14 @@ public:
     gms::feature tablets { *this, "TABLETS"sv };
     gms::feature uuid_sstable_identifiers { *this, "UUID_SSTABLE_IDENTIFIERS"sv };
     gms::feature table_digest_insensitive_to_expiry { *this, "TABLE_DIGEST_INSENSITIVE_TO_EXPIRY"sv };
+    // If this feature is enabled, schema versions are persisted by the group 0 command
+    // that modifies schema instead of being calculated as a digest (hash) by each node separately.
+    // The feature controls both the 'global' schema version (the one gossiped as application_state::SCHEMA)
+    // and the per-table schema versions (schema::version()).
+    // The feature affects non-Raft mode as well (e.g. during RECOVERY), where we send additional
+    // tombstones and flags to schema tables when performing schema changes, allowing us to
+    // revert to the digest method when necessary (if we must perform a schema change during RECOVERY).
+    gms::feature group0_schema_versioning { *this, "GROUP0_SCHEMA_VERSIONING"sv };
 
     // A feature just for use in tests. It must not be advertised unless
     // the "features_enable_test_feature" injection is enabled.