Merge 'Introduce Encryption-at-Rest (EAR) for sstables and commitlog' from Calle Wilund

Fixes https://github.com/scylladb/scylla-enterprise/issues/5016#issuecomment-2558464631

EAR - encryption at rest. Allows on-disk file encryption of sstables and commitlog data.
Introduces OpenSSL based file level encrypted storage, managed via a set of providers
ranging from local files to cloud KMS providers.

For a more comprehensive explanation, see the included docs (or if possible, original
source tree).

Manual bulk merge of EAR feature from enterprise repo to main scylla repo.

Breaks some features apart, but main EAR is still a humongous commit, because to separate this
I would have to mess with code incrementally, adding time and risk.

This PR includes the local file gen tool, tests and also p11 validation.

Note: CI will not execute the full tests unless master CI is set to provide the same environment
as the enterprise one. Not sure about the status of this ATM.

Note: Includes code to compile against cryptsoft kmipc SDK, but not the SDK. If you happen to
check out this tree in the scylla folder and configure, it will be linked against and KMIP functionality
will be enabled, otherwise not.

Closes scylladb/scylladb#22233

* github.com:scylladb/scylladb:
  docs: Add EAR docs
  main/build: Add p11-kit and initialize
  tools: Add local-file-key-generator tool
  tests: Add EAR tests
  tmpdir: shorten test tempdir path
  EAR: port the ear feature from enterprise
  cql_test_env: Add optional query timeout
  schema/migration_manager: Add schema validate
  sstables: add get_shared_components accessor
  config/config_file: Add exports and definitions of config_type_for<>

CMakeLists.txt

@@ -148,6 +148,7 @@ find_package(ICU COMPONENTS uc i18n REQUIRED)
 find_package(fmt 10.0.0 REQUIRED)
 find_package(libdeflate REQUIRED)
 find_package(libxcrypt REQUIRED)
+find_package(p11-kit REQUIRED)
 find_package(Snappy REQUIRED)
 find_package(RapidJSON REQUIRED)
 find_package(xxHash REQUIRED)
@@ -309,6 +310,7 @@ set(scylla_libs
     cql3
     data_dictionary
     dht
+    encryption
     gms
     idl
     index
@@ -345,6 +347,7 @@ if(Scylla_ENABLE_LTO)
 endif()
 
 target_link_libraries(scylla PRIVATE
+    p11-kit::p11-kit
     Seastar::seastar
     absl::headers
     yaml-cpp::yaml-cpp

cmake/Findkmip.cmake

@@ -0,0 +1,53 @@
+#
+# Copyright 2024-present ScyllaDB
+#
+
+#
+# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+#
+
+set(kmip_ver "2.1.0t")
+
+cmake_host_system_information(
+  RESULT distrib_id QUERY DISTRIB_ID)
+if(distrib_id MATCHES "centos|fedora|rhel")
+  set(kmip_distrib "rhel84")
+else()
+  message(FATAL_ERROR "Could not locate kmipc library for ${distrib_id}")
+endif()
+
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+  set(kmip_arch "aarch64")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
+  set(kmip_arch "64")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(powerpc|ppc)64le")
+  set(kmip_arch "ppc64le")
+endif()
+
+set(kmip_ROOT "${PROJECT_SOURCE_DIR}/kmipc/kmipc-${kmip_ver}-${kmip_distrib}_${kmip_arch}")
+find_library(kmip_LIBRARY
+  NAMES kmip
+  HINTS ${kmip_ROOT}/lib)
+
+find_path(kmip_INCLUDE_DIR
+  NAMES kmip.h
+  HINTS ${kmip_ROOT}/include)
+
+mark_as_advanced(
+  kmip_LIBRARY
+  kmip_INCLUDE_DIR)
+
+find_package_handle_standard_args(kmip
+  REQUIRED_VARS
+    kmip_LIBRARY
+    kmip_INCLUDE_DIR)
+
+if(kmip_FOUND)
+  if (NOT TARGET KMIP::kmipc)
+    add_library(KMIP::kmipc UNKNOWN IMPORTED)
+    set_target_properties(KMIP::kmipc PROPERTIES
+      INTERFACE_INCLUDE_DIRECTORIES "${kmip_INCLUDE_DIR}"
+      IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+      IMPORTED_LOCATION "${kmip_LIBRARY}")
+  endif()
+endif()

cmake/Findp11-kit.cmake

@@ -0,0 +1,48 @@
+#
+# Copyright 2023-present ScyllaDB
+#
+
+#
+# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+#
+find_package(PkgConfig REQUIRED)
+
+pkg_check_modules(PC_p11_kit QUIET p11-kit-1)
+
+find_library(p11-kit_LIBRARY
+  NAMES p11-kit
+  PATH_SUFFIXES p11-kit-1
+  HINTS
+    ${PC_p11_kit_LIBDIR}
+    ${PC_p11_kit_LIBRARY_DIRS})
+
+find_path(p11-kit_INCLUDE_DIR
+  NAMES p11-kit/p11-kit.h
+  HINTS
+    ${PC_p11_kit_INCLUDEDIR}
+    ${PC_p11_kit_INCLUDE_DIRS})
+
+mark_as_advanced(
+  p11-kit_LIBRARY
+  p11-kit_INCLUDE_DIR)
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(p11-kit
+  REQUIRED_VARS
+    p11-kit_LIBRARY
+    p11-kit_INCLUDE_DIR
+  VERSION_VAR PC_p11_kit_VERSION)
+
+if(p11-kit_FOUND)
+  set(p11-kit_LIBRARIES ${p11-kit_LIBRARY})
+  set(p11-kit_INCLUDE_DIRS ${p11-kit_INCLUDE_DIR})
+  if(NOT(TARGET p11-kit::p11-kit))
+    add_library(p11-kit::p11-kit UNKNOWN IMPORTED)
+
+    set_target_properties(p11-kit::p11-kit
+      PROPERTIES
+        IMPORTED_LOCATION ${p11-kit_LIBRARY}
+        INTERFACE_INCLUDE_DIRECTORIES ${p11-kit_INCLUDE_DIRS})
+  endif()
+endif()

configure.py

@@ -481,6 +481,8 @@ scylla_tests = set([
     'test/boost/double_decker_test',
     'test/boost/duration_test',
     'test/boost/dynamic_bitset_test',
+    'test/boost/encrypted_file_test',
+    'test/boost/encryption_at_rest_test',
     'test/boost/enum_option_test',
     'test/boost/enum_set_test',
     'test/boost/estimated_histogram_test',
@@ -560,6 +562,7 @@ scylla_tests = set([
     'test/boost/token_metadata_test',
     'test/boost/top_k_test',
     'test/boost/transport_test',
+    'test/boost/symmetric_key_test',
     'test/boost/types_test',
     'test/boost/utf8_test',
     'test/boost/vint_serialization_test',
@@ -1131,6 +1134,19 @@ scylla_core = (['message/messaging_service.cc',
                 'utils/arch/powerpc/crc32-vpmsum/crc32_wrapper.cc',
                 'querier.cc',
                 'mutation_writer/multishard_writer.cc',
+                'ent/encryption/encryption_config.cc',
+                'ent/encryption/encryption.cc',
+                'ent/encryption/symmetric_key.cc',
+                'ent/encryption/local_file_provider.cc',
+                'ent/encryption/replicated_key_provider.cc',
+                'ent/encryption/system_key.cc',
+                'ent/encryption/encrypted_file_impl.cc',
+                'ent/encryption/kmip_host.cc',
+                'ent/encryption/kmip_key_provider.cc',
+                'ent/encryption/kms_host.cc',
+                'ent/encryption/kms_key_provider.cc',
+                'ent/encryption/gcp_host.cc',
+                'ent/encryption/gcp_key_provider.cc',
                 'ent/ldap/ldap_connection.cc',
                 'multishard_mutation_query.cc',
                 'reader_concurrency_semaphore.cc',
@@ -1330,7 +1346,8 @@ scylla_tests_dependencies = scylla_core + alternator + idls + scylla_tests_gener
 
 scylla_raft_dependencies = scylla_raft_core + ['utils/uuid.cc', 'utils/error_injection.cc', 'utils/exceptions.cc']
 
-scylla_tools = ['tools/read_mutation.cc',
+scylla_tools = ['tools/scylla-local-file-key-generator.cc',
+                'tools/read_mutation.cc',
                 'tools/scylla-types.cc',
                 'tools/scylla-sstable.cc',
                 'tools/scylla-nodetool.cc',
@@ -1997,7 +2014,6 @@ def query_seastar_flags(pc_file, use_shared_libs, link_static_cxx=False):
         libs = f"-Wl,-rpath='{rpath}' {libs}"
     if link_static_cxx:
         libs = libs.replace('-lstdc++ ', '')
-
     testing_libs = pkg_config(pc_file.replace('seastar.pc', 'seastar-testing.pc'), '--libs', '--static')
     return {'seastar_cflags': cflags,
             'seastar_libs': libs,
@@ -2010,7 +2026,7 @@ pkgs = ['libsystemd',
 pkgs.append('lua53' if have_pkg('lua53') else 'lua')
 
 
-libs = ' '.join([maybe_static(args.staticyamlcpp, '-lyaml-cpp'), '-latomic', '-lz', '-lsnappy',
+libs = ' '.join([maybe_static(args.staticyamlcpp, '-lyaml-cpp'), '-latomic', '-lz', '-lsnappy', '-lcrypto',
                  ' -lstdc++fs', ' -lcrypt', ' -lcryptopp', ' -lpthread', ' -lldap -llber',
                  # Must link with static version of libzstd, since
                  # experimental APIs that we use are only present there.
@@ -2021,6 +2037,8 @@ libs = ' '.join([maybe_static(args.staticyamlcpp, '-lyaml-cpp'), '-latomic', '-l
                  '-ldeflate',
                 ])
 
+args.user_cflags += " " + pkg_config('p11-kit-1', '--cflags')
+
 if not args.staticboost:
     user_cflags += ' -DBOOST_ALL_DYN_LINK'
 
@@ -2032,6 +2050,32 @@ user_ldflags += ' -fvisibility=hidden'
 if args.staticcxx:
     user_ldflags += " -static-libstdc++"
 
+kmip_lib_ver = '1.9.2a';
+
+def kmiplib():
+    os_ids = get_os_ids()
+    for id in os_ids:
+        if id in { 'centos', 'fedora', 'rhel' }:
+            return 'rhel84'
+    print('Could not resolve libkmip.a for platform {}'.format(os_ids))
+    sys.exit(1)
+
+def target_cpu():
+    cpu, _, _ = subprocess.check_output([cxx, '-dumpmachine']).decode('utf-8').partition('-')
+    return cpu    
+
+def kmip_arch():
+    arch = target_cpu()
+    if arch == 'x86_64':
+        return '64'
+    return arch 
+
+kmipc_dir = f'kmipc/kmipc-2.1.0t-{kmiplib()}_{kmip_arch()}'
+kmipc_lib = f'{kmipc_dir}/lib/libkmip.a'
+libs += ' -lboost_filesystem'
+if os.path.exists(kmipc_lib):
+    libs += f' {kmipc_lib}'
+    user_cflags += f' -I{kmipc_dir}/include -DHAVE_KMIP'
 
 def get_extra_cxxflags(mode, mode_config, cxx, debuginfo):
     cxxflags = []

db/config.cc

@@ -144,6 +144,9 @@ const config_type config_type_for<std::string> = config_type("string", value_to_
 template <>
 const config_type config_type_for<std::vector<sstring>> = config_type("string list", value_to_json<std::vector<sstring>>);
 
+template <>
+const config_type config_type_for<std::unordered_map<sstring, std::unordered_map<sstring, sstring>>> = config_type("string map map", value_to_json<std::unordered_map<sstring, std::unordered_map<sstring, sstring>>>);
+
 template <>
 const config_type config_type_for<std::unordered_map<sstring, sstring>> = config_type("string map", value_to_json<std::unordered_map<sstring, sstring>>);
 

docs/dev/file_encryption.md

@@ -0,0 +1,76 @@
+File level encryption in scylla enterprise
+==========================================
+
+File encryption in scylla enterprise is done by "block-level" encryption via a `file_impl` implementation that transparently wraps file IO transforming data
+to/from encrypted state. Refer to `encrypted_file_impl` in `ent/encryption/encrypted_file_impl.cc`.
+
+Encryption is algorithm-agnostic in that the wrapper supports any symmetric-key algorithm (block cipher) that is available in the OpenSSL EVP (envelope)
+library.
+
+The wrapper uses a user-provided symmetric key coupled with ESSIV block initialization vector calculation. *NOTE*: the data file itself does *not* keep track of
+the key used to encrypt data, thus an external meta data provider is required to map files to their keys, and is solely the user's responsibility.
+
+File block encryption does not use padding, since it relies on input data size and output data size being identical.
+
+The file is divided in `N` blocks of 4096 bytes size. Each 4KB block is encrypted with the provided key `K`, configured block cipher `B`, and block cipher
+operating mode (usually CBC). Because 4KB is an integral multiple of any considerable block cipher's block size, no padding is necessary within any 4KB file
+block.
+
+The initialization vector (IV) for the block cipher operating mode (usually CBC) of each 4KB file block is derived via
+[ESSIV](https://en.wikipedia.org/wiki/Disk_encryption_theory#Encrypted_salt-sector_initialization_vector_(ESSIV)):
+
+- The user-provided data encryption key `K` is hashed with SHA256 to value `h` (32 bytes = 256 bits).
+
+- An AES256 block cipher is keyed with `h`.
+
+- For the particular file block number, a byte array is populated with 8 `NUL` bytes, followed by the little-endian representation of the `uint64_t` block
+  number (16 bytes = 128 bits).
+
+- The 16 byte array is encrypted with a single round (i.e., one ECB mode application) of the AES256 block cipher, to value `c` (16 bytes = 128 bits).
+
+- `c` is the IV of the block cipher `B` (truncated or zero-padded as required by the block size of `B`).
+
+```
+h := SHA256(K)
+IV_B(block_number) := AES256_h(uint64_t(0) ‖ block_number_le64)
+```
+
+
+Padding/truncation
+==================
+
+All encryption is done unpadded. To handle file sizes we use a simplified padding scheme:
+
+Since all writes are assumed to be done by us, and must be aligned (scylla requirement), we can assume in turn that any resizing should be made by truncation.
+
+If a file is truncated to a size that is not a whole multiple of the `B` block cipher's block size (which is typically 16 bytes = 128 bits), then we increment
+the actual truncation size by `B`'s block size.
+
+
+```
+                     +----------- 16 bytes ----------+
+                     |                               |
+               +----------- 16 bytes ----------+     |
+               |     |                         |     |
+               v     v                         v     v
++--------------+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+| n * 16 bytes |  T  |            P            | T'  |
++--------------+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+                     ^                               ^
+                     |                               |
+requested truncation offset: n * 16 + 3              |
+                                                     |
+            actual truncation offset: (n + 1) * 16 + 3
+
+```
+
+- This preserves, in its entirety, for decryption's sake, the ciphertext block (`T ‖ P`) that the user expects to be truncated.
+
+- It records the useful byte count (`size(T)`) of the final ciphertext block (`T ‖ P`) through the trailing misalignment (`size(T')`). The contents of `T'` are
+  irrelevant (it's a partial cipher block, so it cannot be decrypted); only its size matters.
+
+When reading an encrypted file, we check the file size. If we're reading from a file with an unaliged size (i.e., `size(T')` is nonzero), we know that the size
+of the padding at the end is `B`'s block size. `T'` is discarded; the last complete cipher block (`T ‖ P`) is decrypted. After decryption, `P` is discarded as
+well. File size query methods adjust the returned values accordingly.
+
+Non-empty files that are shorter than `B`'s block size are invalid -- they can never be created by the above-described padding scheme.

docs/operating-scylla/_common/tools_index.rst

@@ -8,6 +8,8 @@
 * :doc:`cassandra-stress </operating-scylla/admin-tools/cassandra-stress/>` A tool for benchmarking and load testing a ScyllaDB and Cassandra clusters.
 * :doc:`SSTabledump </operating-scylla/admin-tools/sstabledump>`
 * :doc:`SSTableMetadata </operating-scylla/admin-tools/sstablemetadata>`
+* configuration_encryptor - :doc:`encrypt at rest </operating-scylla/security/encryption-at-rest>` sensitive scylla configuration entries using system key.
+* scylla local-file-key-generator - Generate a local file (system) key for :doc:`encryption at rest </operating-scylla/security/encryption-at-rest>`, with the provided length, Key algorithm, Algorithm block mode and Algorithm padding method.
 * `scyllatop <https://www.scylladb.com/2016/03/22/scyllatop/>`_ - A terminal base top-like tool for scylladb collectd/prometheus metrics.
 * :doc:`scylla_dev_mode_setup</getting-started/installation-common/dev-mod>` - run ScyllaDB in Developer Mode.
 * :doc:`perftune</operating-scylla/admin-tools/perftune>` - performance configuration.

docs/operating-scylla/nodetool-commands/upgradesstables.rst

@@ -55,6 +55,7 @@ Additional References
 ---------------------
 
 .. include:: nodetool-index.rst
+:doc:`Encryption at Rest </operating-scylla/security/encryption-at-rest>`
 
 
 

docs/operating-scylla/procedures/backup-restore/restore.rst

@@ -35,6 +35,10 @@ Procedure
 Repeat the following steps for each node in the cluster:
 --------------------------------------------------------
 
+.. note::
+
+   If you are restoring :doc:`encrypted backup files </operating-scylla/security/encryption-at-rest>`, make sure Scylla Enterprise has the same keys used by Scylla to encrypt the data before starting the restore process.
+
 .. note::
 
       Best practise is **not** to restore :doc:`Materialized Views (MV) </features/materialized-views>` and :doc:`Secondary Indexes (SI) </features/secondary-indexes>` SSTables.

docs/operating-scylla/procedures/cassandra-to-scylla-migration-process.rst

@@ -51,6 +51,19 @@ Procedure
    - Import schema to ScyllaDB: ``cqlsh [IP] --file 'adjusted_schema.cql'``
 
 .. _`limitations and known issues section`: #notes-limitations-and-known-issues
+
+
+.. note::
+
+   Scylla and Apache Cassandra :doc:`encrypted backup files </operating-scylla/security/encryption-at-rest>` are **not** compatible.
+   sstableloader does **not** support loading from encrypted files.
+
+   If you need to migrate/restore from encrypted files:
+
+   * Upload them to the original database
+   * Decrypted the table with ALTER TABLE
+   * Update the SSTables files with :doc:`upgradesstable </operating-scylla/nodetool-commands/upgradesstables>`
+   * Use sstableloader
                                              
 .. note::
 

docs/operating-scylla/security/_common/security-index.rst

@@ -8,6 +8,7 @@
 * :doc:`Encryption: Data in Transit Client to Node </operating-scylla/security/client-node-encryption/>`
 * :doc:`Encryption: Data in Transit Node to Node </operating-scylla/security/node-node-encryption/>`
 * :doc:`Generating a self-signed Certificate Chain Using openssl </operating-scylla/security/generate-certificate/>`
+* :doc:`Encryption at Rest </operating-scylla/security/encryption-at-rest>`
 
 
 

docs/operating-scylla/security/encryption-at-rest.rst

@@ -0,0 +1,859 @@
+==================
+Encryption at Rest
+==================
+
+Introduction
+----------------------
+
+ScyllaDB Enterprise protects your sensitive data with data-at-rest encryption. 
+It protects the privacy of your user's data, reduces the risk of data breaches, and helps meet regulatory requirements. 
+In particular, it provides an additional level of protection for your data persisted in storage or its backups.
+
+When ScyllaDB Enterprise Encryption at Rest is used together with Encryption in Transit (:doc:`Node to Node </operating-scylla/security/node-node-encryption>`  and :doc:`Client to Node </operating-scylla/security/client-node-encryption>`), you benefit from end to end data encryption.
+
+About Encryption at Rest
+-----------------------------
+
+The following can be encrypted:
+
+* ScyllaDB persistent tables (SSTables)
+* System level data, such as:
+
+  - Commit logs
+  - Batches
+  - hints logs
+  - KMIP Password (part of scylla.yaml)
+
+Encryption at Rest works at table level granularity, so you can choose to encrypt only sensitive tables. For both system and table data, you can use different algorithms that are supported by `OpenSSL <https://www.openssl.org/>`_ in a file block encryption scheme. 
+
+.. note:: SSTables of a particular table can have different encryption keys, use different encryption algorithms, or not be encrypted at all - at the same time. 
+
+When is Data Encrypted?
+========================
+
+As SSTables are immutable, tables are encrypted only once, as a result of memtable flush, compaction, or upgrade (with :doc:`Nodetool upgradesstables </operating-scylla/nodetool-commands/upgradesstables>`).
+
+Once a table is encrypted, all resulting SSTables are encrypted using the most current key and algorithm. 
+When you encrypt an existing table, the new SSTables are encrypted. The old SSTables which existed before the encryption are not updated. These tables are encrypted according to the same actions as described previously. 
+
+When is Data Decrypted?
+========================
+
+When ScyllaDB reads an encrypted SSTable from disk, it fetches the encryption key's ID from the SSTable and uses it to extract the key and decrypt the data.
+When ScyllaDB reads an encrypted system table, it fetches the system table encryption key location from the scylla.yaml file. It locates the key and uses it to extract the key and decrypt the data. 
+
+
+Encryption Key Types
+----------------------
+
+Two types of encryption keys are available: System Keys and Table Keys. 
+
+System Keys
+====================
+
+System keys are used for encrypting system data, such as commit logs, hints, and/or other user table keys. When a Replicated Key Provider is used for encrypting SSTables, the table keys are stored in the encrypted_keys table, and the system key is used to encrypt the encrypted_keys table. The system key is stored as the contents of a local file and is encrypted with a single key that you provide. The default location of system keys is ``/etc/scylla/resources/system_keys/`` and can be changed with the ``system_key_directory`` option in scylla.yaml file. When a Local Key Provider is used for encrypting system info, you can provide your own key, or ScyllaDB can make one for you.
+
+.. _Replicated:
+
+Table Keys
+===================
+Table keys are used for encrypting SSTables. Depending on your key provider, this key is stored in different locations:
+
+* Replicated Key Provider - encrypted_keys table
+* KMIP Key Provider - KMIP server
+* KMS Key Provider - AWS
+* Local Key Provider - in a local file with multiple keys. You can provide your own key or ScyllaDB can make one for you.
+
+.. _ear-key-providers:
+
+.. note::
+
+   Encrypted SStables undergo a regular backup procedure. Ensure you keep your 
+   encryption key available in case you need to restore from backup.
+
+Key Providers
+----------------------
+
+When encrypting the system tables or SSTables, you need to state which provider is holding your keys. You can use the following options:
+
+.. list-table::
+   :widths: 33 33 33
+   :header-rows: 1
+
+   * - Key Provider Name
+     - key_provider Name
+     - Description 
+   * - Local Key Provider 
+     - LocalFileSystemKeyProviderFactory (**default**)
+     - Stores the key on the same machine as the data.   
+   * - Replicated Key Provider 
+     - ReplicatedKeyProviderFactory
+     - Stores table keys in a ScyllaDB table where the table itself is encrypted using the system key (available from 2019.1.3)
+   * - KMIP Key Provider 
+     - KmipKeyProviderFactory
+     - External key management server (available from 2019.1.3)
+   * - KMS Key Provider
+     - KmsKeyProviderFactory
+     - Uses key(s) provided by the AWS KMS service.
+   * - GCP Key Provider
+     - GcpKeyProviderFactory
+     - Used key(s) provided by the GCP KMS service.
+
+
+About Local Key Storage
+==========================
+
+Local keys are used for encrypting user data, such as SSTables. 
+Currently, this is the only option  available for user data and, as such, is the default key storage manager. 
+With local key storage, keys are stored locally on disk in a text file. The location of this file is specified in the scylla.yaml. 
+
+.. caution:: Care should be taken so that no unauthorized person can access the key data from the file system. Make sure that the owner of this file is the ``scylla`` user and that the file is **not** readable by **other users**, not accessible by **other roles**. 
+
+You should also consider keeping the key directory on a network drive (using TLS for the file sharing) to avoid having keys and data on the same storage media, in case your storage is stolen or discarded. 
+
+.. _ear-cipher-algorithms:
+
+Cipher Algorithms
+----------------------
+
+The following cipher_algorithims are available for use with ScyllaDB using `OpenSSL <https://www.openssl.org/>`_. Note that the default algorithm (AES/CBC/PKCS5Padding with key strength 128 ) is recommended.
+
+.. list-table::
+   :widths: 70 30
+   :header-rows: 1
+
+   * - cipher_algorithm 
+     - secret_key_strength
+   * - AES/CBC/PKCS5Padding (**default**)
+     - 128 (**default**), 192, or 256 
+   * - AES/ECB/PKCS5Padding
+     - 128, 192, or 256
+   * - Blowfish/CBC/PKCS5Padding    
+     - 32-448
+
+.. _ear-create-encryption-key:
+
+Create Encryption Keys
+-----------------------------
+
+Depending on your key provider, you will either have the option of allowing ScyllaDB to generate an encryption key, or you will have to provide one:
+
+* KMIP Key Provider - you don't need to generate any key yourself
+* KMS Key Provider - you must generate a key yourself in AWS
+* Replicated Key Provider - you must generate a system key yourself
+* Local Key Provider - If you do not generate your own secret key, ScyllaDB will create one for you
+
+When encrypting ScyllaDB config by ``configuration_encryptor``, you also need to generate a secret key and upload the key to all nodes.
+
+
+Use the key generator script
+================================
+
+The Key Generator script generates a key in the directory of your choice. 
+
+**Procedure**
+
+
+#. Create (if it doesn't exist) a local directory for storing the key.  Make sure that the owner of the directory is ``scylla`` and not another user. Make sure that the ``scylla`` user can read, write, and execute over the parent directory. Following this procedure makes ``/etc/scylla/encryption_keys/`` the parent directory of your keys.
+
+   For example:
+
+   .. code-block:: none
+
+      sudo mkdir -p /etc/scylla/encryption_keys/system_keys
+      sudo chown -R scylla:scylla /etc/scylla/encryption_keys
+      sudo chmod -R 700 /etc/scylla/encryption_keys
+
+#. Create a key using the local file key generator script making sure that the keyfile owner is ``scylla`` and not another user. Run the command:
+
+   .. code-block:: none
+
+      sudo -u scylla /usr/bin/scylla local-file-key-generator <op> [options] [key-path]
+
+   Where:
+
+   * ``-a,--alg <arg>`` - the encryption algorithm (e.g., AES) you want to use to encrypt the key
+   * ``-h,--help`` - displays the help menu
+   * ``-l,--length <arg>`` - the length of the encryption key in bits (i.e. 128, 256)
+   * ``-b,--block-mode <arg>`` - the encryption algorithm block mode (i.e. CBC, EBC)
+   * ``-p,--padding <arg>`` - the encryption algorithm padding method (i.e. PKCS5)
+   * ``key-path`` - is the directory you want to place the key into (/etc/scylla/encryption_keys, for example)
+
+   And ``<op>`` is one of ``generate`` or ``append``, the first creating a new key file with the generated key, the latter
+   appending a new key of the required type to an existing file.
+
+   For Example:
+
+   To create a secret key and a system key using other encryption settings in a different location:
+
+   .. code-block:: none
+
+      sudo -u scylla /usr/bin/scylla local-file-key-generator generate -a AES -b ECB -p PKCS5 -l 192 /etc/scylla/encryption_keys/secret_key
+      sudo -u scylla /usr/bin/scylla local-file-key-generator generate -a AES -b CBC -p PKCS5 -l 128 /etc/scylla/encryption_keys/system_keys/system_key
+   
+   To display the secret key parameters:
+
+   .. code-block:: none
+
+      sudo cat /etc/scylla/encryption_keys/secret_key
+
+   Returns:
+   
+   .. code-block:: none
+   
+      AES/ECB/PKCS5Padding:192:8stVxW5ypYhNxsnRVS1A6suKhk0sG4Tj
+
+   To display the system key parameters:
+   
+   .. code-block:: none
+   
+      sudo cat /etc/scylla/encryption_keys/system_keys/system_key
+
+   Returns:
+   
+   .. code-block:: none
+   
+      AES/CBC/PKCS5Padding:128:GGpOSxTGhtPRPLrNPYvVMQ==
+
+
+   Once you have created a key, copy the key to each node, using the procedure described in `Copy keys to nodes`_.
+
+Copy keys to nodes
+======================
+
+Every key you generate needs to be copied to the nodes for use in local key providers. 
+
+**Procedure**
+
+#. Securely copy the key file, using ``scp`` or similar, to the same path on all nodes in the cluster. Make sure the key on each target node is moved to the same location as the source directory and that the target directory has the same permissions as the source directory. 
+
+#. Repeat for all nodes in the cluster.
+
+.. _encryption-at-rest-set-kmip:
+
+Set the KMIP Host
+----------------------
+
+If you are using :term:`KMIP <Key Management Interoperability Protocol (KMIP)>` to encrypt tables or system information, add the KMIP server information to the ``scylla.yaml`` configuration file. 
+
+#. Edit the ``scylla.yaml`` file located in ``/etc/scylla/`` and add the following in KMIP host(s) section:
+
+   .. code-block:: yaml
+
+      #
+      # kmip_hosts:
+      #   <name>:
+      #       hosts: <address1[:port]> [, <address2[:port]>...]
+      #       certificate: <identifying certificate> (optional)
+      #       keyfile: <identifying key> (optional; it is required if "certificate" is set)
+      #       truststore: <truststore for SSL connection> (optional)
+      #       certficate_revocation_list: <CRL file> (optional)
+      #       priority_string: <kmip tls priority string> 
+      #       username: <login> (optional>
+      #       password: <password> (optional)
+      #       max_command_retries: <int> (optional; default 3)
+      #       key_cache_expiry: <key cache expiry period>
+      #       key_cache_refresh: <key cache refresh/prune period>
+      #   <name>:
+   
+   Where:
+
+   * ``<name>`` - The cluster name.
+   * ``hosts`` - The list of hosts specified by IP and port for the KMIP server. The KMIP connection management only supports failover, so all requests go through a single KMIP server. There is no load balancing, as currently no KMIP servers support read replication or other strategies for availability. Hosts are tried in the order they appear, and the next one in the list is tried if the previous one fails. The default number of retries is three, but you can customize it with "max_command_retries". 
+   * ``certificate`` - The name of the certificate and path used to identify yourself to the KMIP server.
+   * ``keyfile`` - The name of the key used to identify yourself to the KMIP server. It is generated together with the certificate.
+   * ``truststore`` - The location and key for the truststore to present to the KMIP server.
+   * ``certficate_revocation_list`` - The path to a PEM-encoded certificate revocation list (CRL) - a list of issued certificates that have been revoked before their expiration date.
+   * ``priority_string`` - The KMIP TLS priority string.
+   * ``username`` - The KMIP server user name.
+   * ``password`` - The KMIP server password.
+   * ``max_command_retries`` - The number of attempts to connect to the KMIP server before trying the next host in the list. 
+   * ``key_cache_expiry`` - Key cache expiry period, after which keys will be re-requested from server. Default is 600s.
+   * ``key_cache_refresh`` - Key cache refresh period - the frequency at which cache is checked for expired entries. Default is 1200s.   
+
+#. Save the file. 
+#. Drain the node with :doc:`nodetool drain </operating-scylla/nodetool-commands/drain>`
+#. Restart the scylla-server service.
+
+.. include:: /rst_include/scylla-commands-restart-index.rst
+ 
+.. _encryption-at-rest-set-kms:
+
+Set the KMS Host
+----------------------
+
+.. note:: KMS support is available since ScyllaDB Enterprise **2023.1.1**.
+
+If you are using AWS KMS to encrypt tables or system information, add the KMS information to the ``scylla.yaml`` configuration file. 
+
+#. Edit the ``scylla.yaml`` file located in ``/etc/scylla/`` to add the following in KMS host(s) section:
+
+   .. code-block:: yaml
+
+       kms_hosts:
+         <name>:
+             endpoint: http(s)://<host>(:port) (optional if `aws_region` is specified)
+             aws_region: <aws region> (optional if `endpoint` is specified)
+             aws_access_key_id: <aws access key id> (optional)
+             aws_secret_access_key: <aws secret access key> (optional)
+             aws_profile: <aws credentials profile to use> (optional)
+             aws_use_ec2_credentials: (bool : default false)
+             aws_use_ec2_region: (bool : default false)
+             aws_assume_role_arn: <arn of aws role to assume before call> (optional)
+             master_key: <named KMS key for encrypting data keys> (required)
+             certificate: <identifying certificate> (optional)
+             keyfile: <identifying key> (optional)
+             truststore: <truststore for SSL connection> (optional)
+             priority_string: <KMS TLS priority string> (optional)
+             key_cache_expiry: <key cache expiry period>
+             key_cache_refresh: <key cache refresh/prune period>
+      #   <name>:
+   
+   Where:
+
+   * ``<name>`` - The name to identify the KMS host. You have to provide this name to encrypt a :ref:`new <ear-create-table>` or :ref:`existing <ear-alter-table>` table.
+   * ``endpoint`` - The explicit KMS host endpoint. If not provided, ``aws_region`` is used for connection.
+   * ``aws_region`` - An AWS region. If not provided, ``endpoint`` is used for connection.
+   * ``aws_access_key_id`` - AWS access key used for authentication. If not specified, the provider reads it from your AWS credentials.
+   * ``aws_secret_access_key`` - AWS secret access key used for authentication. If not specified, the provider reads it from your AWS credentials.
+   * ``aws_profile`` - AWS profile to use if reading credentials from file
+   * ``aws_use_ec2_credentials`` - If true, KMS queries will use the credentials provided by ec2 instance role metadata as initial access key.
+   * ``aws_use_ec2_region`` - If true, KMS queries will use the AWS region indicated by ec2 instance metadata.
+   * ``aws_assume_role_arn`` - If set, any KMS query will first attempt to assume this role. 
+   * ``master_key`` - The ID or alias of your AWS KMS key. The key must be generated with an appropriate access policy so that the AWS user has permissions to read the key and encrypt data using that key. This parameter is required.
+   * ``certificate`` - The name of the certificate and the path used to identify yourself to the KMS server.
+   * ``keyfile`` - The name of the key for the certificate. It is generated together with the certificate.
+   * ``truststore`` - The location and key for the truststore to present to the KMS server.
+   * ``priority_string`` - The KMS TLS priority string.
+   * ``key_cache_expiry`` - Key cache expiry period, after which keys will be re-requested from server. Default is 600s.
+   * ``key_cache_refresh`` - Key cache refresh period - the frequency at which cache is checked for expired entries. Default is 1200s.   
+
+   .. note::
+
+      Note that either ``endpoint``, ``aws_region`` or ``aws_use_ec2_region`` must be set (one of them is required for connection).
+
+   Example:
+
+   .. code-block:: yaml
+
+       kms_hosts:
+         my-kms1:
+             aws_use_ec2_credentials: true
+             aws_use_ec2_region: true
+             master_key: myorg/MyKey
+
+#. Save the file. 
+#. Drain the node with :doc:`nodetool drain </operating-scylla/nodetool-commands/drain>`
+#. Restart the scylla-server service.
+
+.. include:: /rst_include/scylla-commands-restart-index.rst
+
+.. _encryption-at-rest-set-gcp:
+
+Set the GCP Host
+----------------------
+
+If you are using Google GCP KMS to encrypt tables or system information, add the GCP information to the ``scylla.yaml`` configuration file. 
+
+#. Edit the ``scylla.yaml`` file located in ``/etc/scylla/`` to add the following in KMS host(s) section:
+
+   .. code-block:: yaml
+
+       gcp_hosts:
+         <name>:
+             gcp_project_id: <gcp project>
+             gcp_location: <gcp location>
+             gcp_credentials_file: <(service) account json key file - authentication>
+             gcp_impersonate_service_account: <service account to impersonate>
+             master_key: <keyring>/<keyname> - named GCP key for encrypting data keys (required)
+             certificate: <identifying certificate> (optional)
+             keyfile: <identifying key> (optional)
+             truststore: <truststore for SSL connection> (optional)
+             priority_string: <KMS TLS priority string> (optional)
+             key_cache_expiry: <key cache expiry period>
+             key_cache_refresh: <key cache refresh/prune period>
+      #   <name>:
+   
+   Where:
+
+   * ``<name>`` - The name to identify the GCP host. You have to provide this name to encrypt a :ref:`new <ear-create-table>` or :ref:`existing <ear-alter-table>` table.
+   * ``gcp_project_id`` - The GCP project from which to retrieve key information.
+   * ``gcp_location`` - A GCP project location.
+   * ``gcp_credentials_file`` - GCP credentials file used for authentication. If not specified, the provider reads it from your GCP credentials.
+   * ``gcp_impersonate_service_account`` - An optional service account to impersonate when issuing key query calls.
+   * ``master_key`` - The <keyring>/<keyname> of your GCP KMS key. The key must be generated with an appropriate access policy so that the AWS user has permissions to read the key and encrypt data using that key. This parameter is required.
+   * ``certificate`` - The name of the certificate and the path used to identify yourself to the KMS server.
+   * ``keyfile`` - The name of the key for the certificate. It is generated together with the certificate.
+   * ``truststore`` - The location and key for the truststore to present to the KMS server.
+   * ``priority_string`` - The KMS TLS priority string.
+   * ``key_cache_expiry`` - Key cache expiry period, after which keys will be re-requested from server. Default is 600s.
+   * ``key_cache_refresh`` - Key cache refresh period - the frequency at which cache is checked for expired entries. Default is 1200s.   
+
+   Example:
+
+   .. code-block:: yaml
+
+       gcp_hosts:
+         my-gcp1:
+             gcp_project_id: myproject
+             gcp_location: global
+             master_key: mykeyring/mykey
+
+#. Save the file. 
+#. Drain the node with :doc:`nodetool drain </operating-scylla/nodetool-commands/drain>`
+#. Restart the scylla-server service.
+
+.. include:: /rst_include/scylla-commands-restart-index.rst
+
+Encrypt Tables
+-----------------------------
+
+.. note::
+
+   This feature is available since ScyllaDB Enterprise 2023.1.2.
+
+ScyllaDB allows you to enable or disable default encryption of tables. 
+When enabled, tables will be encrypted by default using the configuration 
+provided for the ``user_info_encryption`` option in the ``scylla.yaml`` file.
+
+You can override the default configuration when you CREATE TABLE or ALTER TABLE
+with ``scylla_encryption_options``. See :ref:`Encrypt a Single Table <ear-create-table>`
+for details.
+
+**Before you Begin**
+
+Ensure you have an encryption key available:
+
+* If you are using AWS KMS, :ref:`set the KMS Host <encryption-at-rest-set-kms>`.
+* If you are using KMIP, :ref:`set the KMIP Host <encryption-at-rest-set-kmip>`.
+* If you are using Google GCP KMS, :ref:`set the GCP Host <encryption-at-rest-set-gcp>`.
+* If you want to create your own key, follow the procedure in :ref:`Create Encryption Keys <ear-create-encryption-key>`.
+* If you do not create your own key, use the following procedure for ScyllaDB 
+  to create a key for you (the default location ``/etc/scylla/data_encryption_keys`` may cause 
+  permission issues; the following example creates a key in the directory ``/etc/scylla/encryption_keys``):
+
+   .. code-block:: none
+   
+      sudo mkdir -p /etc/scylla/encryption_keys
+      sudo chown -R scylla:scylla /etc/scylla/encryption_keys
+      sudo chmod -R 700 /etc/scylla/encryption_keys
+
+**Procedure**
+
+Edit the ``scylla.yaml`` file located in ``/etc/scylla/`` and configure 
+the ``user_info_encryption`` option:
+
+.. code-block:: yaml
+   
+   user_info_encryption:
+    enabled: <true|false>
+    cipher_algorithm: <hashing algorithm to create the key>
+    secret_key_strength: <length of the key> 
+    key_provider: <your key provider>
+    secret_key_file: <key file>
+    kmip_host: <your kmip_host>
+    kms_host: <your kms_host>
+    gcp_host: <your gcp_host>
+
+Where:
+
+* ``enabled`` - Enables or disables default table encryption. Required.
+* ``cipher_algorithm`` - One of the :ref:`cipher algorithms <ear-cipher-algorithms>`. 
+  If not provided, the default will be used.
+* ``secret_key_strength`` - The length of the key in bytes ( determined by 
+  the :ref:`cipher algorithms <ear-cipher-algorithms>` you choose).
+  If not provided, the default will be used.
+* ``key_provider`` - The name of the key provider. See :ref:`Key Providers <ear-key-providers>`. 
+  Required.
+* ``secret_key_file`` - The location of the key created by ScyllaDB (by default ``/etc/scylla/data_encryption_keys``).
+  Required if you use a ScyllaDB-generated key.
+* ``kmip_host`` - The name of your :ref:`kmip_host <encryption-at-rest-set-kmip>` group.
+  Required if you use KMIP.
+* ``kms_host`` - The name of your :ref:`kms_host <encryption-at-rest-set-kms>` group.
+  Required if you use KMS.
+* ``gcp_host`` - The name of your :ref:`gcp_host <encryption-at-rest-set-gcp>` group.
+  Required if you use GCP.
+
+**Example**
+
+.. code-block:: yaml
+   
+   user_info_encryption:
+    enabled: true
+    cipher_algorithm: AES
+    secret_key_strength: 128  
+    key_provider: LocalFileSystemKeyProviderFactory
+    secret_key_file: scylla /etc/scylla/encryption_keys
+
+**Examples for KMS:**
+
+In the following example, the ``master_key`` configured for :ref:`kms_host <encryption-at-rest-set-kms>` will be used.
+
+.. code-block:: yaml
+
+   user_info_encryption:
+    enabled: true
+    key_provider: KmsKeyProviderFactory
+    kms_host: my-kms1
+   
+
+You can specify a different ``master_key`` than the one configured for :ref:`kms_host <encryption-at-rest-set-kms>`:
+
+ .. code-block:: yaml
+   
+   user_info_encryption:
+    enabled: true
+    key_provider: KmsKeyProviderFactory
+    kms_host: my-kms1
+    master_key: myorg/SomeOtherKey
+
+.. _ear-create-table:
+
+Encrypt a Single Table
+-----------------------------
+
+This procedure demonstrates how to encrypt a new table.
+
+**Before you Begin**
+
+* Make sure to `Set the KMIP Host`_ if you are using KMIP, or the the :ref:`KMS Host <encryption-at-rest-set-kms>` if you are using AWS KMS.
+
+* If you want to make your own key, use the procedure in `Create Encryption Keys`_ and skip to step 3. If you do not create your own key, ScyllaDB will create one for you in the ``secret_key_file`` path. If you are not creating your own key, start with step 1.  
+
+**Procedure**
+
+#. By default, the encryption key is located in the ``/etc/scylla/`` directory, and the file is named ``data_encryption_keys``. If you want to save the key in a different directory, create one. This example will create encryption keys in a different directory (``/etc/scylla/encryption_keys``, for example), which ensures that the owner of this directory is ``scylla`` and not another user.
+
+   .. note:: Using the default location results in a known permission issue (scylladb/scylla-tools-java#94), so it is recommended to use another location as described in the example.
+
+   .. code-block:: none
+   
+      sudo mkdir -p /etc/scylla/encryption_keys
+      sudo chown -R scylla:scylla /etc/scylla/encryption_keys
+      sudo chmod -R 700 /etc/scylla/encryption_keys
+
+#. Create the keyspace if it doesn’t exist.
+
+#. Create the table using the ``CREATE TABLE`` CQL statement, adding any :ref:`additional options <create-table-statement>`. To encrypt the table, use the options for encryption below, remembering to set the ``secret_key_file <path>`` to the same directory you created in step 1.
+
+   .. code-block:: cql
+
+      CREATE TABLE <keyspace>.<table_name> (...<columns>...) WITH 
+        scylla_encryption_options = { 
+          'cipher_algorithm' : <hash>,   
+          'secret_key_strength' : <len>,   
+          'key_provider': <provider>, 
+          'secret_key_file': <path> 
+        }
+      ;
+
+   Where:
+
+   * ``cipher_algorithm`` -  The hashing algorithm which is to be used to create the key. See `Cipher Algorithms`_ for more information.
+   * ``secret_key_strength`` - The length of the key in bytes. This is determined by the cipher you choose. See `Cipher Algorithms`_ for more information.
+   * ``key_provider`` is the name or type of key provider. Refer to `Key Providers`_ for more information.
+   * ``secret_key_file`` - the location that ScyllaDB will store the key it creates (if one does not exist in this location) or the location of the key. By default the location is ``/etc/scylla/data_encryption_keys``. 
+
+   **Example:**
+
+   Continuing the example from above, this command will instruct ScyllaDB to encrypt the table and will save the key in the location created in step 1. 
+
+   .. code-block:: cql
+
+      CREATE TABLE data.atrest (pk text primary key, c0 int) WITH 
+        scylla_encryption_options = {  
+          'cipher_algorithm' : 'AES/ECB/PKCS5Padding', 
+          'secret_key_strength' : 128,  
+          'key_provider': 'LocalFileSystemKeyProviderFactory',  
+          'secret_key_file': '/etc/scylla/encryption_keys/data_encryption_keys'
+        }
+      ;
+
+   **Example for KMS:**
+
+   .. code-block:: cql
+
+      CREATE TABLE myks.mytable (...<columns>...) WITH 
+        scylla_encryption_options = { 
+          'cipher_algorithm' :  'AES/CBC/PKCS5Padding',   
+          'secret_key_strength' : 128,   
+          'key_provider': 'KmsKeyProviderFactory', 
+          'kms_host': 'my-kms1'
+        }
+      ;
+
+   You can skip ``cipher_algorithm`` and ``secret_key_strength`` (the :ref:`defaults <ear-cipher-algorithms>` will be used):
+
+   .. code-block:: cql
+
+      CREATE TABLE myks.mytable (...<columns>...) WITH 
+        scylla_encryption_options = { 
+          'key_provider': 'KmsKeyProviderFactory', 
+          'kms_host': 'my-kms1'
+        }
+      ;
+
+   You can specify a different master key than the one configured for ``kms_host`` in the ``scylla.yaml`` file:
+
+   .. code-block:: cql
+
+      CREATE TABLE myks.mytable (...<columns>...) WITH 
+        scylla_encryption_options = { 
+          'key_provider': 'KmsKeyProviderFactory', 
+          'kms_host': 'my-kms1',
+          'master_key':'myorg/SomeOtherKey'
+        }
+      ;
+
+
+#. From this point, every new SSTable created for the ``atrest`` table is encrypted, using the ``data_encryption_keys`` key located in ``/etc/scylla/encryption_keys/``. This table will remain encrypted with this key until you either change the key, change the key properties, or disable encryption.
+
+#. To ensure all SSTables for this table on every node are encrypted, run the :doc:`Nodetool upgradesstables </operating-scylla/nodetool-commands/upgradesstables>` command. If not, the SSTables remain unencrypted until they are compacted or flushed from MemTables. 
+
+   For Example:
+
+   .. code-block:: none
+
+      nodetool upgradesstables data atrest
+
+#. Your SSTables are encrypted. If you want to change the key at any point, use the `Update Encryption Properties of Existing Tables`_ procedure. Always keep your key in a safe location known to you. Do not lose it. See `When a Key is Lost`_.
+
+.. _ear-alter-table:
+
+Update Encryption Properties of Existing Tables
+==================================================
+
+You can encrypt any existing table or use this procedure to change the cipher algorithm, key location or key strength or even disable encryption on a table. 
+
+**Procedure**
+
+#. Edit the table properties to enable encryption of one table of your choosing. Use the properties explained in `Encrypt a Single Table`_ if needed. 
+
+   .. code-block:: cql
+
+      ALTER TABLE <keyspace>.<table_name> (...<columns>...) WITH 
+        scylla_encryption_options = { 
+          'cipher_algorithm' : <hash>,   
+          'secret_key_strength' : <len>,   
+          'key_provider': <provider>, 
+          'secret_key_file': <path> 
+        }
+      ;
+
+
+   **Example:**
+
+   Continuing the example from above, this command will instruct ScyllaDB to encrypt the table and will save the key in the location created in step 1. 
+
+   .. code-block:: cql
+
+      ALTER TABLE data.atrest (pk text primary key, c0 int) WITH 
+        scylla_encryption_options = {  
+          'cipher_algorithm' : 'AES/ECB/PKCS5Padding', 
+          'secret_key_strength' : 192,  
+          'key_provider': 'LocalFileSystemKeyProviderFactory',  
+          'secret_key_file': '/etc/scylla/encryption_keys/data_encryption_keys'
+        }
+      ;
+   
+   **Example for KMS:**
+
+   .. code-block:: cql
+
+      ALTER TABLE myks.mytable (...<columns>...) WITH 
+        scylla_encryption_options = { 
+          'cipher_algorithm' :  'AES/CBC/PKCS5Padding',   
+          'secret_key_strength' : 128,   
+          'key_provider': 'KmsKeyProviderFactory', 
+          'kms_host': 'my-kms1'
+        }
+      ;
+
+#. If you want to make sure that SSTables that existed before this change are also encrypted, you can either upgrade them using the ``nodetool upgradesstables`` command or wait until the next compaction. If you decide to wait, ScyllaDB will still be able to read the old unencrypted tables. If you change the key or remove encryption, ScyllaDB will still continue to read the old tables as long as you still have the key. If your data is encrypted and you do not have the key, your data is unreadable. 
+
+   * If you decide to upgrade all of your old SSTables run the :doc:`nodetool upgradesstables </operating-scylla/nodetool-commands/upgradesstables>` command. 
+
+     .. code-block:: none
+
+         nodetool upgradesstables <keyspace> <table>
+
+     For example:
+
+     .. code-block:: none
+
+         nodetool upgradesstables ks test  
+
+   * Repeat this command on all nodes as nodetool runs locally. 
+ 
+#. If you want to change the key or disable encryption, repeat the `Update Encryption Properties of Existing Tables`_ procedure using the examples below as reference. 
+
+**Examples**
+
+To encrypt an existing table named test in keyspace ks:
+
+.. code-block:: cql
+  
+   ALTER TABLE ks.test WITH
+     scylla_encryption_options = { 
+        'cipher_algorithm' : 'AES/ECB/PKCS5Padding', 
+        'secret_key_strength' : 128,  
+        'key_provider': 'LocalFileSystemKeyProviderFactory',  
+        'secret_key_file': '/etc/scylla/encryption_keys/data_encryption_keys'
+     }
+   ;
+
+
+To change the cipher algorithm from AES/ECB/PKCS5Padding to AES/ECB/PKCS5Padding and to change the key strength from 128 to 192 on an existing table:
+
+.. code-block:: cql
+  
+   ALTER TABLE ks.test WITH
+     scylla_encryption_options = { 
+        'cipher_algorithm' : 'AES/ECB/PKCS5Padding', 
+        'secret_key_strength' : 192,  
+        'key_provider': 'LocalFileSystemKeyProviderFactory',  
+        'secret_key_file': '/etc/scylla/encryption_keys/data_encryption_keys'
+     }
+   ;
+
+To disable encryption on an encrypted table named test in keyspace ks:
+
+.. code-block:: cql
+
+   ALTER TABLE ks.test WITH
+      scylla_encryption_options =  { 'key_provider' : 'none’ };
+
+
+Encrypt System Resources
+---------------------------
+
+System encryption is applied to semi-transient on-disk data, such as commit logs, batch logs, and hinted handoff data. 
+This ensures that all temporarily stored data is encrypted until fully persisted to final SSTable on disk. 
+Once this encryption is enabled, it is used for all system data. 
+
+
+**Procedure**
+
+#. Edit the scylla.yaml file - located in /etc/scylla/scylla.yaml and add the following:
+
+   .. code-block:: none
+
+      system_info_encryption:
+         enabled: <true|false>
+         key_provider: (optional) <key provider type>
+         system_key_directory: <path to location of system key>
+
+   Where: 
+
+   * ``enabled`` can be true or false. True is enabled; false is disabled.  
+
+   * ``key_provider`` is the name or type of key provider. Refer to `Key Providers`_ for more information.
+
+   * ``cipher_algorithm`` is one of the supported `Cipher Algorithms`_.
+
+   * ``secret_key_file`` is the name of the key file containing the secret key (key.pem, for example)
+
+   Example:
+
+   .. code-block:: none
+
+      system_info_encryption:
+         enabled: True
+         cipher_algorithm: AES
+         secret_key_strength: 128  
+         key_provider: LocalFileSystemKeyProviderFactory
+         secret_key_file: /path/to/systemKey.pem
+
+   Example for KMIP:
+
+   .. code-block:: none
+
+      system_info_encryption:
+         enabled: True
+         cipher_algorithm: AES
+         secret_key_strength: 128  
+         key_provider: KmipKeyProviderFactory
+         kmip_host:  yourkmipServerIP.com
+
+   Where ``kmip_host`` is the address for your KMIP server.
+
+   Example for KMS:
+
+   .. code-block:: none
+
+      system_info_encryption:
+         enabled: True
+         cipher_algorithm: AES/CBC/PKCS5Padding
+         secret_key_strength: 128  
+         key_provider: KmsKeyProviderFactory
+         kms_host: myScylla
+
+   Where ``kms_host`` is the unique name of the KMS host specified in the scylla.yaml file.
+
+   Example for GCP:
+
+   .. code-block:: none
+
+      system_info_encryption:
+         enabled: True
+         cipher_algorithm: AES/CBC/PKCS5Padding
+         secret_key_strength: 128  
+         key_provider: GcpKeyProviderFactory
+         gcp_host: myScylla
+
+   Where ``gcp_host`` is the unique name of the GCP host specified in the scylla.yaml file.
+
+
+#. Do not close the yaml file. Change the system key directory location according to your settings. 
+
+   * ``system_key_directory`` is the location of the system key you created in `Create Encryption Keys`_.
+
+   .. code-block:: none
+
+      system_key_directory: /etc/scylla/encryption_keys/system_keys
+
+#. Save the file. 
+#. Drain the node with :doc:`nodetool drain </operating-scylla/nodetool-commands/drain>`
+#. Restart the scylla-server service.
+
+   .. include:: /rst_include/scylla-commands-restart-index.rst 
+
+   .. wasn't able to test this successfully
+
+.. Encrypt and Decrypt Configuration Files
+.. =======================================
+
+.. Using the Configuration Encryption tool, you can encrypt parts of the scylla.yaml file which contain encryption configuration settings. 
+
+.. **Procedure**
+
+.. 1.  Run the Configuration Encryption script:
+
+.. test code-block: none
+
+.. /bin/configuration_encryptor [options] [key-path]
+
+.. Where:
+
+.. * ``-c, --config`` - the path to the configuration file (/etc/scylla/scylla.yaml, for example)
+.. * ``-d, --decrypt`` - decrypts the configuration file at the specified path
+.. * ``-o, --output`` - (optional) writes the configuration file to a specified target. This can be the same location as the source file. 
+.. * ``-h. --help`` - help for this command
+
+.. For example:
+
+.. test code-block: none
+
+.. sudo -u scylla /bin/configuration_encryptor -c /etc/scylla/scylla.yaml /etc/scylla/encryption_keys/secret_key
+.. end of test
+
+When a Key is Lost
+----------------------
+
+It is crucial to back up all of your encryption keys in a secure way. Keep a copy of all keys in a secure location. In the event that you do lose a key, your data encrypted with that key will be unreadable.
+
+Additional Resources
+----------------------
+
+* :doc:`nodetool upgradesstables </operating-scylla/nodetool-commands/upgradesstables>`
+* :ref:`CREATE TABLE parameters <create-table-statement>`

docs/operating-scylla/security/index.rst

@@ -18,6 +18,7 @@ Security
    node-node-encryption
    generate-certificate
    saslauthd
+   encryption-at-rest
    ldap-authentication
    ldap-authorization
 
@@ -51,6 +52,6 @@ Security
   * :doc:`Encryption: Data in Transit Client to Node </operating-scylla/security/client-node-encryption/>`
   * :doc:`Encryption: Data in Transit Node to Node </operating-scylla/security/node-node-encryption/>`
   * :doc:`Generating a self-signed Certificate Chain Using openssl </operating-scylla/security/generate-certificate/>`
-  * `Encryption at Rest <https://enterprise.docs.scylladb.com/stable/operating-scylla/security/encryption-at-rest.html>`_ available in `ScyllaDB Enterprise <https://enterprise.docs.scylladb.com/>`_
+  * :doc:`Encryption at Rest </operating-scylla/security/encryption-at-rest>`
 
 Also check out the `Security Features lesson <https://university.scylladb.com/courses/scylla-operations/lessons/security-features/topic/security-features/>`_ on ScyllaDB University.

docs/operating-scylla/security/security-checklist.rst

@@ -43,12 +43,14 @@ Configure ScyllaDB to use TLS/SSL for all the connections. Use TLS/SSL to encryp
 
 Encryption at Rest
 ~~~~~~~~~~~~~~~~~~
-Encryption at Rest is available in `ScyllaDB Enterprise <https://enterprise.docs.scylladb.com/>`_.
+Encryption at Rest is available in a Scylla Enterprise 2019.1.1.
 
 Encryption at Rest protects the privacy of your user's data, reduces the risk of data breaches, and helps meet regulatory requirements. 
 In particular, it provides an additional level of protection for your data persisted in storage or backup.
 
-See `Encryption at Rest <https://enterprise.docs.scylladb.com/stable/operating-scylla/security/encryption-at-rest.html>`_ for details.
+See:
+
+* :doc:`Encryption at Rest </operating-scylla/security/encryption-at-rest>`
 
 Reduce the Network Exposure
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/reference/glossary.rst

@@ -58,6 +58,9 @@ Glossary
     Keyspace
       A collection of tables with attributes which define how data is replicated on nodes. See :doc:`Ring Architecture </architecture/ringarchitecture/index>`.
     
+    Key Management Interoperability Protocol (KMIP)
+      :abbr:`KMIP (Key Management Interoperability Protocol)` is a communication protocol that defines message formats for storing keys on a key management server (KMIP server). You can use a KMIP server to protect your keys when using Encryption at Rest. See :doc:`Encryption at Rest</operating-scylla/security/encryption-at-rest/>`.
+
     Leveled compaction strategy (LCS)
       :abbr:`LCS (Leveled compaction strategy)` uses small, fixed-size (by default 160 MB) SSTables divided into different levels. See :doc:`Compaction Strategies</architecture/compaction/compaction-strategies/>`.
 

ent/CMakeLists.txt

@@ -1 +1,2 @@
+add_subdirectory(encryption)
 add_subdirectory(ldap)

ent/encryption/CMakeLists.txt

@@ -0,0 +1,44 @@
+include(add_whole_archive)
+
+find_package(cpp-jwt REQUIRED)
+find_package(kmip)
+
+add_library(scylla_encryption STATIC)
+target_sources(scylla_encryption
+  PRIVATE
+    encrypted_file_impl.cc
+    encryption.cc
+    encryption_config.cc
+    gcp_host.cc
+    gcp_key_provider.cc
+    kmip_host.cc
+    kmip_key_provider.cc
+    kms_host.cc
+    kms_key_provider.cc
+    local_file_provider.cc
+    replicated_key_provider.cc
+    symmetric_key.cc
+    system_key.cc)
+target_include_directories(scylla_encryption
+  PUBLIC
+    ${CMAKE_SOURCE_DIR})
+target_link_libraries(scylla_encryption
+  PUBLIC
+    Seastar::seastar
+  PRIVATE
+    cql3
+    utils
+    cpp-jwt::cpp-jwt)
+if(kmip_FOUND)
+  target_link_libraries(scylla_encryption
+    PRIVATE
+      KMIP::kmipc)
+  target_compile_definitions(scylla_encryption
+    PUBLIC
+      HAVE_KMIP)
+endif()
+
+check_headers(check-headers scylla_encryption
+  GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh)
+
+add_whole_archive(encryption scylla_encryption)

ent/encryption/encrypted_file_impl.cc

@@ -0,0 +1,555 @@
+/*
+ * Copyright (C) 2018 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#include <fcntl.h>
+#include <fmt/format.h>
+#include <seastar/core/file.hh>
+#include <seastar/core/byteorder.hh>
+
+#include "symmetric_key.hh"
+#include "encryption.hh"
+#include "utils/serialization.hh"
+#include "encrypted_file_impl.hh"
+
+namespace encryption {
+
+static inline bool is_aligned(size_t n, size_t a) {
+    return (n & (a - 1)) == 0;
+}
+
+/**
+ * Very simple block-encrypting file wrapper.
+ *
+ * Uses user provided symmetric key + ESSIV block IV calculation
+ * to encrypt data.
+ *
+ * The essiv block key is created by generating the SHA256 hash
+ * of the provided data encryption key bytes, truncated to block_key_len/8
+ * and generating an AES/ECB key using this data.
+ *
+ * The file is divided in N blocks of `block_size` size.
+ * Each block is encrypted (unpadded) with the provided key and
+ * block mode, using an IV derived by (essiv):
+ *
+ *  bytes tmp[<key block size, typically 16>] = { 0, ..., uint64_t-little-endian(<block number>) }
+ *  iv = block_key->encrypt(tmp);
+ *
+ * All encryption is done unpadded. To handle file sizes we use
+ * a slightly shaky scheme:
+ *
+ * Since all writes are assumed to be done by us, and must be aligned,
+ * we can assume in turn that any resizing should be made by our truncate
+ * method. If we attept to truncate to a size not a multiple of our
+ * _key_ block size (typically 16), we add the same size to the actual
+ * truncation size.
+ * On read we then check the file size. If we're reading from a file
+ * with unaliged size, we know there are key-block-size junk at the end.
+ * We can align down the last decryption call to match block size, then
+ * discard the excessive bytes from the result.
+ *
+ * If we're in a read/write situation, we need to keep size updated, and
+ * we could possibly race with disk op/continuations.
+ * But we are really only for ro/wo cases.
+ */
+class encrypted_file_impl : public seastar::file_impl {
+    file _file;
+    ::shared_ptr<symmetric_key> _key;
+    ::shared_ptr<symmetric_key> _block_key;
+    bytes _hash_salt;
+
+    std::optional<uint64_t> _file_length;
+
+    // this is somewhat large, but we assume this is for bulky stuff like sstables/commitlog
+    // so large alignment should be preferable to reaclculating block IV too often.
+    static constexpr size_t block_size = 4096;
+    static constexpr size_t block_key_len = 256;
+
+    class my_file_handle_impl;
+    friend class my_file_handle_impl;
+
+    bytes iv_for(uint64_t pos) const;
+
+    using mode = symmetric_key::mode;
+
+    temporary_buffer<uint8_t> transform(uint64_t, const void* buffer, size_t len, mode);
+    size_t transform(uint64_t, const void* buffer, size_t len, void*, mode);
+
+    future<> verify_file_length();
+    void maybe_set_length(uint64_t);
+    void clear_length();
+
+    static ::shared_ptr<symmetric_key> generate_block_key(::shared_ptr<symmetric_key>);
+
+public:
+    encrypted_file_impl(file, ::shared_ptr<symmetric_key>);
+
+    future<size_t> write_dma(uint64_t pos, const void* buffer, size_t len, io_intent*) override;
+    future<size_t> write_dma(uint64_t pos, std::vector<iovec> iov, io_intent*) override;
+    future<size_t> read_dma(uint64_t pos, void* buffer, size_t len, io_intent*) override;
+    future<size_t> read_dma(uint64_t pos, std::vector<iovec> iov, io_intent*) override;
+
+
+    future<> flush() override {
+        return _file.flush();
+    }
+    future<struct stat> stat(void) override;
+    future<> truncate(uint64_t length) override;
+    future<> discard(uint64_t offset, uint64_t length) override {
+        return _file.discard(offset, length);
+    }
+    future<> allocate(uint64_t position, uint64_t length) override {
+        return _file.allocate(position, length);
+    }
+    future<uint64_t> size(void) override;
+    future<> close() override {
+        return _file.close();
+    }
+    std::unique_ptr<file_handle_impl> dup() override;
+
+    subscription<directory_entry> list_directory(std::function<future<> (directory_entry de)> next) override {
+        return _file.list_directory(std::move(next));
+    }
+    future<temporary_buffer<uint8_t>> dma_read_bulk(uint64_t offset, size_t range_size, io_intent*) override;
+};
+
+/**
+ * Note: ESSIV block iv generation implementation.
+ * See: http://securityevaluators.com/knowledge/papers/fde_whitepaper_draft_20170627.pdf
+ *
+ * We generate a key based on the sha256 of the data key, then encrypt each block number
+ * using this to get per-block IV.
+ * The key is AES-256, using ECB (non-iv) encryption
+ *
+ */
+::shared_ptr<symmetric_key> encrypted_file_impl::generate_block_key(::shared_ptr<symmetric_key> key) {
+    auto hash = calculate_sha256(key->key());
+    hash.resize(block_key_len / 8);
+    return ::make_shared<symmetric_key>(key_info{"AES/ECB", block_key_len }, hash);
+}
+
+encrypted_file_impl::encrypted_file_impl(file f, ::shared_ptr<symmetric_key> key)
+    : _file(std::move(f))
+    , _key(std::move(key))
+    , _block_key(generate_block_key(_key))
+{
+    _memory_dma_alignment = std::max<unsigned>(_file.memory_dma_alignment(), block_size);
+    _disk_read_dma_alignment = std::max<unsigned>(_file.disk_read_dma_alignment(), block_size);
+    _disk_write_dma_alignment = std::max<unsigned>(_file.disk_write_dma_alignment(), block_size);
+}
+
+static future<uint64_t> calculate_file_length(const file& f, size_t key_block_size) {
+    return f.size().then([key_block_size](uint64_t s) {
+        if (!is_aligned(s, key_block_size)) {
+            if (s < key_block_size) {
+                throw std::domain_error(fmt::format("file size {}, expected 0 or at least {}", s, key_block_size));
+            }
+            s -= key_block_size;
+        }
+        return s;
+    });
+}
+
+future<> encrypted_file_impl::verify_file_length() {
+    if (_file_length) {
+        return make_ready_future();
+    }
+    return calculate_file_length(_file, _key->block_size()).then([this](uint64_t s) {
+        _file_length = s;
+    });
+}
+
+void encrypted_file_impl::maybe_set_length(uint64_t s) {
+    if (s > _file_length.value_or(0)) {
+        _file_length = s;
+    }
+}
+
+void encrypted_file_impl::clear_length() {
+    _file_length = std::nullopt;
+}
+
+bytes encrypted_file_impl::iv_for(uint64_t pos) const {
+    assert(!(pos & (block_size - 1)));
+
+    // #658. ECB block mode has no IV. Bad for security,
+    // but must handle.
+    size_t iv_len = _key->iv_len();
+    if (iv_len == 0) {
+        return bytes{};
+    }
+
+    assert(iv_len >= _key->block_size());
+    assert(iv_len >= sizeof(uint64_t));
+
+    bytes b(bytes::initialized_later(), std::max(iv_len, _block_key->block_size()));
+    std::fill(b.begin(), b.end() - sizeof(uint64_t), 0);
+
+    // write block pos as little endian IV-len integer
+    auto block = pos / block_size;
+    write_le(reinterpret_cast<char *>(b.end()) - sizeof(uint64_t), block);
+
+    // encrypt the encoded block number to build an IV
+    _block_key->encrypt_unpadded(b.data(), b.size(), b.data());
+
+    b.resize(iv_len);
+
+    return b;
+}
+
+size_t encrypted_file_impl::transform(uint64_t pos, const void* buffer, size_t len, void* dst, mode m) {
+    assert(!(pos & (block_size - 1)));
+    assert(_file_length || m == mode::encrypt);
+
+    auto o = reinterpret_cast<char*>(dst);
+    auto i = reinterpret_cast<const char*>(buffer);
+    auto l = _file_length.value_or(std::numeric_limits<uint64_t>::max());
+    auto b = _key->block_size();
+
+    size_t off = 0;
+
+    for (; off < len; off += block_size) {
+        auto iv = iv_for(pos + off);
+        auto rem = std::min<uint64_t>(block_size, len - off);
+
+        if (rem < block_size || ((pos + off + rem) > l && m == symmetric_key::mode::decrypt)) {
+            // truncated block. should be the last one.
+            if (m != symmetric_key::mode::decrypt) {
+                throw std::invalid_argument("Output data not aligned");
+            }
+            _key->transform_unpadded(m, i + off, align_down(rem, b), o + off, iv.data());
+            return l - pos;
+        }
+        _key->transform_unpadded(m, i + off, block_size, o + off, iv.data());
+    }
+
+    return off;
+}
+
+temporary_buffer<uint8_t> encrypted_file_impl::transform(uint64_t pos, const void* buffer, size_t len, mode m) {
+    assert(!(len & (block_size - 1)));
+    auto tmp = temporary_buffer<uint8_t>::aligned(_file.memory_dma_alignment(), len);
+    auto s = transform(pos, buffer, len, tmp.get_write(), m);
+    tmp.trim(s);
+    return tmp;
+}
+
+future<size_t> encrypted_file_impl::write_dma(uint64_t pos, const void* buffer, size_t len, io_intent* intent) {
+    assert(!(len & (block_size - 1)));
+    auto tmp = transform(pos, buffer, len, mode::encrypt);
+    assert(tmp.size() == len); // writing
+    auto p = tmp.get();
+    return _file.dma_write(pos, p, len, intent).then([this, tmp = std::move(tmp), pos](size_t s) {
+        maybe_set_length(pos + s);
+        return s;
+    });
+}
+
+future<size_t> encrypted_file_impl::write_dma(uint64_t pos, std::vector<iovec> iov, io_intent* intent) {
+    std::vector<temporary_buffer<uint8_t>> tmp;
+    tmp.reserve(iov.size());
+    size_t n = 0;
+    for (auto& i : iov) {
+        assert(!(i.iov_len & (block_size - 1)));
+
+        tmp.emplace_back(transform(pos + n, i.iov_base, i.iov_len, mode::encrypt));
+        assert(tmp.back().size() == i.iov_len); // writing
+        n += i.iov_len;
+        i = iovec{ tmp.back().get_write(), tmp.back().size() };
+    }
+    return _file.dma_write(pos, std::move(iov), intent).then([this, tmp = std::move(tmp), pos](size_t s) {
+        maybe_set_length(pos + s);
+        return s;
+    });
+}
+
+future<size_t> encrypted_file_impl::read_dma(uint64_t pos, void* buffer, size_t len, io_intent* intent) {
+    assert(!(len & (block_size - 1)));
+    return verify_file_length().then([this, pos, buffer, len, intent] {
+        return _file.dma_read(pos, buffer, len, intent).then([this, pos, buffer](size_t len) {
+            return transform(pos, buffer, len, buffer, mode::decrypt);
+        });
+    });
+}
+
+future<size_t> encrypted_file_impl::read_dma(uint64_t pos, std::vector<iovec> iov, io_intent* intent) {
+    return verify_file_length().then([this, pos, iov = std::move(iov), intent]() mutable {
+        auto f = _file.dma_read(pos, iov, intent);
+        return f.then([this, pos, iov = std::move(iov)](size_t len) mutable {
+            size_t off = 0;
+            for (auto& i : iov) {
+                off += transform(pos + off, i.iov_base, i.iov_len, i.iov_base, mode::decrypt);
+            }
+            return off;
+        });
+    });
+}
+
+future<temporary_buffer<uint8_t>> encrypted_file_impl::dma_read_bulk(uint64_t offset, size_t range_size, io_intent* intent) {
+    return verify_file_length().then([this, offset, range_size, intent]() mutable {
+        auto front = offset & (block_size - 1);
+        offset -= front;
+        range_size += front;
+        // enterprise #925
+        // If caller is clever and asks for the last chunk of file
+        // explicitly (as in offset = N, range_size = size() - N),
+        // or any other unaligned size, we need to add enough padding
+        // to get the actual full block to decode.
+        auto block_size = align_up(range_size, _key->block_size());
+        return _file.dma_read_bulk<uint8_t>(offset, block_size, intent).then([this, offset, front, range_size](temporary_buffer<uint8_t> result) {
+            auto s = transform(offset, result.get(), result.size(), result.get_write(), mode::decrypt);
+            // never give back more than asked for.
+            result.trim(std::min(s, range_size));
+            result.trim_front(front);
+            return result;
+        });
+    });
+}
+
+future<> encrypted_file_impl::truncate(uint64_t length) {
+    return size().then([this, length](uint64_t s) {
+        if (s >= length) {
+            auto kb = _key->block_size();
+            auto n = length;
+            if (!is_aligned(length, kb)) {
+                n += kb;
+            }
+            return _file.truncate(n).then([this, length] {
+                _file_length = length;
+            });
+        }
+
+        // crap. we need to pad zeros. But zeros here means
+        // encrypted zeros. So we must do this surprisingly
+        // expensively, by actually writing said zeros block
+        // by block. Anyone hoping for sparse files is now
+        // severely disappointed!
+
+        auto buf_size = align_up(std::min(length, 32 * block_size), block_size);
+        auto aligned_size = align_down(s, block_size);
+
+        temporary_buffer<char> buf(buf_size);
+        std::fill(buf.get_write(), buf.get_write() + buf_size, 0);
+
+        struct trunc {
+            temporary_buffer<char> buf;
+            uint64_t aligned_size;
+            uint64_t size;
+            uint64_t length;
+        };
+
+        return do_with(trunc{std::move(buf), aligned_size, s, length}, [this](trunc & t) {
+            return repeat([this, &t] {
+                if (t.aligned_size >= t.length) {
+                    return make_ready_future<stop_iteration>(stop_iteration::yes);
+                }
+                auto n = std::min(t.buf.size(), align_up(size_t(t.length - t.aligned_size), block_size));
+                if (t.aligned_size < t.size) {
+                    return read_dma(t.aligned_size, t.buf.get_write(), n, nullptr).then([&, n](size_t r) mutable {
+                        auto rem = size_t(t.size - t.aligned_size);
+                        auto ar = align_up(r, block_size);
+                        assert(ar <= t.buf.size());
+                        if (rem < ar) {
+                            std::fill(t.buf.get_write() + rem, t.buf.get_write() + ar, 0);
+                        }
+                        return write_dma(t.aligned_size, t.buf.get(), ar, nullptr).then([&, n](size_t w) {
+                            t.aligned_size += w;
+                            // #1869. On btrfs, we get the buffer potentially clobbered up to "n" (max read amount)
+                            // even when "r" (actual bytes read) is less.
+                            std::fill(t.buf.get_write(), t.buf.get_write() + n, 0);
+                            return make_ready_future<stop_iteration>(stop_iteration::no);
+                        });
+                    });
+                }
+                return write_dma(t.aligned_size, t.buf.get(), n, nullptr).then([&](size_t w) {
+                    t.aligned_size += w;
+                    return make_ready_future<stop_iteration>(stop_iteration::no);
+                });
+            });
+        }).then([this, length] {
+            return truncate(length);
+        });;
+    });
+}
+
+future<struct stat> encrypted_file_impl::stat() {
+    return _file.stat().then([this](struct stat s) {
+        return verify_file_length().then([this, s]() mutable {
+            s.st_size = *_file_length;
+            return s;
+        });
+    });
+}
+
+future<uint64_t> encrypted_file_impl::size() {
+    return  verify_file_length().then([this] {
+        return *_file_length;
+    });
+}
+
+
+std::unique_ptr<file_handle_impl> encrypted_file_impl::dup() {
+    class my_file_handle_impl : public seastar::file_handle_impl {
+        seastar::file_handle _handle;
+        key_info _info;
+        bytes _key;
+    public:
+        my_file_handle_impl(seastar::file_handle h, const key_info& info, const bytes& key)
+            : _handle(std::move(h))
+            , _info(info)
+            , _key(key)
+        {}
+        std::unique_ptr<file_handle_impl> clone() const override {
+            return std::make_unique<my_file_handle_impl>(_handle, _info, _key);
+        }
+        seastar::shared_ptr<file_impl> to_file() && override {
+            return seastar::make_shared<encrypted_file_impl>(_handle.to_file(), ::make_shared<symmetric_key>(_info, _key));
+        }
+    };
+
+    return std::make_unique<my_file_handle_impl>(_file.dup(), _key->info(), _key->key());
+}
+
+shared_ptr<file_impl> make_encrypted_file(file f, ::shared_ptr<symmetric_key> k) {
+    return ::make_shared<encrypted_file_impl>(std::move(f), std::move(k));
+}
+
+class indirect_encrypted_file_impl : public file_impl {
+    ::shared_ptr<file_impl> _impl;
+    file _f;
+    size_t _key_block_size;
+    get_key_func _get;
+
+    future<> get() {
+        if (_impl) {
+           return make_ready_future<>();
+        }
+        return _get().then([this](::shared_ptr<symmetric_key> k) {
+            // #978 could be running the getting more than once.
+            // Only write _impl once though
+            if (!_impl) {
+                _impl = make_encrypted_file(_f, std::move(k));
+            }
+        });
+    }
+public:
+    indirect_encrypted_file_impl(file f, size_t key_block_size, get_key_func get)
+        : _f(f), _key_block_size(key_block_size), _get(std::move(get))
+    {}
+
+    future<size_t> write_dma(uint64_t pos, const void* buffer, size_t len, io_intent* intent) override {
+        return get().then([this, pos, buffer, len, intent]() {
+            return _impl->write_dma(pos, buffer, len, intent);
+        });
+    }
+    future<size_t> write_dma(uint64_t pos, std::vector<iovec> iov, io_intent* intent) override {
+        return get().then([this, pos, iov = std::move(iov), intent]() mutable {
+            return _impl->write_dma(pos, std::move(iov), intent);
+        });
+    }
+    future<size_t> read_dma(uint64_t pos, void* buffer, size_t len, io_intent* intent) override {
+        return get().then([this, pos, buffer, len, intent]() {
+            return _impl->read_dma(pos, buffer, len, intent);
+        });
+    }
+    future<size_t> read_dma(uint64_t pos, std::vector<iovec> iov, io_intent* intent) override {
+        return get().then([this, pos, iov = std::move(iov), intent]() mutable {
+            return _impl->read_dma(pos, std::move(iov), intent);
+        });
+    }
+    future<temporary_buffer<uint8_t>> dma_read_bulk(uint64_t offset, size_t range_size, io_intent* intent) override {
+        return get().then([this, offset, range_size, intent]() {
+            return _impl->dma_read_bulk(offset, range_size, intent);
+        });
+    }
+    future<> flush(void) override {
+        if (_impl) {
+            return _impl->flush();
+        }
+        return _f.flush();
+    }
+    future<struct stat> stat(void) override {
+        if (_impl) {
+            return _impl->stat();
+        }
+        return _f.stat().then([this](struct stat s) {
+           return calculate_file_length(_f, _key_block_size).then([s](uint64_t fs) mutable {
+               s.st_size = fs;
+               return s;
+           });
+        });
+    }
+    future<> truncate(uint64_t length) override {
+        if (_impl) {
+            return _impl->truncate(length);
+        }
+        return _f.truncate(length);
+    }
+    future<> discard(uint64_t offset, uint64_t length) override {
+        if (_impl) {
+            return _impl->discard(offset, length);
+        }
+        return _f.discard(offset, length);
+    }
+    future<> allocate(uint64_t position, uint64_t length) override {
+        if (_impl) {
+            return _impl->allocate(position, length);
+        }
+        return _f.allocate(position, length);
+    }
+    future<uint64_t> size(void) override {
+        if (_impl) {
+            return _impl->size();
+        }
+        return calculate_file_length(_f, _key_block_size);
+    }
+    future<> close() override {
+        if (_impl) {
+            return _impl->close();
+        }
+        return _f.close();
+    }
+    std::unique_ptr<file_handle_impl> dup() override {
+        if (_impl) {
+            return _impl->dup();
+        }
+        class my_file_handle_impl : public seastar::file_handle_impl {
+            seastar::file_handle _handle;
+            size_t _key_block_size;
+            get_key_func _get;
+        public:
+            my_file_handle_impl(seastar::file_handle h, size_t key_block_size, get_key_func get)
+                : _handle(std::move(h))
+                , _key_block_size(key_block_size)
+                , _get(std::move(get))
+            {}
+            std::unique_ptr<file_handle_impl> clone() const override {
+                return std::make_unique<my_file_handle_impl>(_handle, _key_block_size, _get);
+            }
+            seastar::shared_ptr<file_impl> to_file() && override {
+                return make_delayed_encrypted_file(_handle.to_file(), _key_block_size, _get);
+            }
+        };
+        return std::make_unique<my_file_handle_impl>(_f.dup(), _key_block_size, _get);
+    }
+
+    subscription<directory_entry> list_directory(std::function<future<> (directory_entry de)> next) override {
+        if (_impl) {
+            return _impl->list_directory(std::move(next));
+        }
+        return _f.list_directory(std::move(next));
+    }
+};
+
+shared_ptr<seastar::file_impl> make_delayed_encrypted_file(file f, size_t key_block_size, get_key_func get) {
+    return ::make_shared<indirect_encrypted_file_impl>(std::move(f), key_block_size, std::move(get));
+}
+
+
+}
+

ent/encryption/encrypted_file_impl.hh

@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2018 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#include <seastar/core/file.hh>
+#include <seastar/core/shared_ptr.hh>
+
+#include "symmetric_key.hh"
+
+namespace encryption {
+
+class symmetric_key;
+
+shared_ptr<file_impl> make_encrypted_file(file, ::shared_ptr<symmetric_key>);
+
+using get_key_func = std::function<future<::shared_ptr<symmetric_key>>()>;
+
+shared_ptr<file_impl> make_delayed_encrypted_file(file, size_t, get_key_func);
+}

ent/encryption/encryption.hh

@@ -0,0 +1,196 @@
+/*
+ * Copyright (C) 2018 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once
+
+#include <map>
+
+#include <seastar/core/future.hh>
+#include <seastar/core/sstring.hh>
+#include <seastar/core/shared_ptr.hh>
+#include <seastar/core/distributed.hh>
+
+#include <fmt/core.h>
+#include <fmt/ostream.h>
+
+#include "../../bytes.hh"
+#include "../../compress.hh"
+
+class service_set;
+
+namespace replica {
+class database;
+}
+
+namespace db {
+class config;
+class extensions;
+}
+
+namespace cql3 {
+class query_processor;
+}
+namespace service {
+class storage_service;
+class migration_manager;
+}
+
+namespace sstables {
+    class sstable;
+}
+
+namespace encryption {
+inline const sstring KEY_PROVIDER = "key_provider";
+inline const sstring SECRET_KEY_PROVIDER_FACTORY_CLASS = "secret_key_provider_factory_class";
+inline const sstring SECRET_KEY_FILE = "secret_key_file";
+inline const sstring SYSTEM_KEY_FILE = "system_key_file";
+inline const sstring CIPHER_ALGORITHM = "cipher_algorithm";
+inline const sstring SECRET_KEY_STRENGTH = "secret_key_strength";
+
+inline const sstring HOST_NAME = "kmip_host";
+inline const sstring TEMPLATE_NAME = "template_name";
+inline const sstring KEY_NAMESPACE = "key_namespace";
+
+bytes base64_decode(const sstring&, size_t off = 0, size_t n = sstring::npos);
+sstring base64_encode(const bytes&, size_t off = 0, size_t n = bytes::npos);
+bytes calculate_md5(const bytes&, size_t off = 0, size_t n = bytes::npos);
+bytes calculate_sha256(const bytes&, size_t off = 0, size_t n = bytes::npos);
+bytes calculate_sha256(bytes_view);
+bytes hmac_sha256(bytes_view msg, bytes_view key);
+
+future<temporary_buffer<char>> read_text_file_fully(const sstring&);
+future<> write_text_file_fully(const sstring&, temporary_buffer<char>);
+future<> write_text_file_fully(const sstring&, const sstring&);
+
+std::optional<std::chrono::milliseconds> parse_expiry(std::optional<std::string>);
+
+class symmetric_key;
+struct key_info;
+
+using options = std::map<sstring, sstring>;
+using opt_bytes = std::optional<bytes>;
+using key_ptr = shared_ptr<symmetric_key>;
+
+/**
+ * wrapper for "options" (map) to provide an
+ * interface returning empty optionals for
+ * non-available values. Makes query simpler
+ * and allows .value_or(...)-statements, which
+ * are neat for default values.
+ *
+ * In the long run one could contemplate
+ * using non-std maps with similar built-in
+ * functionality for all our various configs
+ * in the system, but for now we are firmly
+ * entrenched in map<string, string>
+ */
+template<typename Map>
+class map_wrapper {
+    const Map& _options;
+public:
+    using mapped_type = typename Map::mapped_type;
+    using key_type = typename Map::key_type;
+
+    map_wrapper(const Map& opts)
+        : _options(opts)
+    {}
+
+    std::optional<mapped_type> operator()(const key_type& k) const {
+        auto i = _options.find(k);
+        if (i != _options.end()) {
+            return i->second;
+        }
+        return std::nullopt;
+    }
+};
+
+using opt_wrapper = map_wrapper<options>;
+
+key_info get_key_info(const options&);
+
+class encryption_context;
+
+class key_provider {
+public:
+    virtual ~key_provider()
+    {}
+    virtual future<std::tuple<key_ptr, opt_bytes>> key(const key_info&, opt_bytes = {}) = 0;
+    virtual future<> validate() const {
+        return make_ready_future<>();
+    }
+    virtual bool should_delay_read(const opt_bytes&) const {
+        return false;
+    }
+private:
+    friend std::ostream& operator<<(std::ostream&, const key_provider&);
+    virtual void print(std::ostream&) const = 0;
+};
+
+std::ostream& operator<<(std::ostream&, const key_provider&);
+
+}
+
+template <> struct fmt::formatter<encryption::key_provider> : fmt::ostream_formatter {};
+
+namespace encryption {
+
+class key_provider_factory {
+public:
+    virtual ~key_provider_factory()
+    {}
+    virtual shared_ptr<key_provider> get_provider(encryption_context& c, const options&) = 0;
+};
+
+class encryption_config;
+class system_key;
+class kmip_host;
+class kms_host;
+class gcp_host;
+
+/**
+ * Context is a singleton object, shared across shards. I.e. even though there are obvious mutating
+ * calls in it, it guarantees thread/shard safety.
+ *
+ * Why is this not a sharded thingamajing? Because its own instance methods need to send itself
+ * as a shard-safe object forwards, and thus need to know that same shard, which breaks the circle of
+ * ownership and stuff.
+ */
+class encryption_context {
+public:
+    virtual ~encryption_context() = default;
+    virtual shared_ptr<key_provider> get_provider(const options&) = 0;
+    virtual shared_ptr<system_key> get_system_key(const sstring&) = 0;
+    virtual shared_ptr<kmip_host> get_kmip_host(const sstring&) = 0;
+    virtual shared_ptr<kms_host> get_kms_host(const sstring&) = 0;
+    virtual shared_ptr<gcp_host> get_gcp_host(const sstring&) = 0;
+
+    virtual shared_ptr<key_provider> get_cached_provider(const sstring& id) const = 0;
+    virtual void cache_provider(const sstring& id, shared_ptr<key_provider>) = 0;
+
+    virtual const encryption_config& config() const = 0;
+    virtual shared_ptr<symmetric_key> get_config_encryption_key() const = 0;
+
+    virtual distributed<cql3::query_processor>& get_query_processor() const = 0;
+    virtual distributed<service::storage_service>& get_storage_service() const = 0;
+    virtual distributed<replica::database>& get_database() const = 0;
+    virtual distributed<service::migration_manager>& get_migration_manager() const = 0;
+
+    sstring maybe_decrypt_config_value(const sstring&) const;
+
+    virtual future<> start() = 0;
+    virtual future<> stop() = 0;
+};
+
+future<seastar::shared_ptr<encryption_context>>
+register_extensions(const db::config&, std::unique_ptr<encryption_config>, db::extensions&, const ::service_set&);
+
+// for testing
+std::string encryption_provider(const sstables::sstable&);
+}
+

ent/encryption/encryption_config.cc

@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2015 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#include <boost/filesystem.hpp>
+
+#include "db/config.hh"
+#include "utils/config_file_impl.hh"
+
+#include "init.hh"
+#include "encryption_config.hh"
+#include "encryption.hh"
+
+#include <fmt/ranges.h>
+
+encryption::encryption_config::encryption_config()
+                : config_file()
+// BEGIN entry definitions
+
+        , system_key_directory(this, "system_key_directory", value_status::Used, "/etc/scylla/conf/resources/system_keys",
+                                R"foo(The directory where system keys are kept
+
+This directory should have 700 permissions and belong to the scylla user)foo")
+
+		, config_encryption_active(this, "config_encryption_active", value_status::Used, false, "")
+
+        , config_encryption_key_name(this, "config_encryption_key_name", value_status::Used, "system_key",
+                                "Set to the local encryption key filename or KMIP key URL to use for configuration file property value decryption")
+
+        , system_info_encryption(this, "system_info_encryption", value_status::Used,
+                                { { "enabled", "false" }, { CIPHER_ALGORITHM,
+                                                "AES/CBC/PKCS5Padding" }, {
+                                                SECRET_KEY_STRENGTH, "128" },
+                                                },
+                                R"foo(System information encryption settings
+
+If enabled, system tables that may contain sensitive information (system.batchlog,
+system.paxos), hints files and commit logs are encrypted with the
+encryption settings below.
+
+When enabling system table encryption on a node with existing data, run
+`nodetool upgradesstables -a` on the listed tables to encrypt existing data.
+
+When tracing is enabled, sensitive info will be written into the tables in the
+system_traces keyspace. Those tables should be configured to encrypt their data
+on disk.
+
+It is recommended to use remote encryption keys from a KMIP server when using 
+Transparent Data Encryption (TDE) features.
+Local key support is provided when a KMIP server is not available.
+
+See the scylla documentation for available key providers and their properties.
+)foo")
+		, kmip_hosts(this, "kmip_hosts", value_status::Used, { },
+                                R"foo(KMIP host(s). 
+
+The unique name of kmip host/cluster that can be referenced in table schema.
+
+host.yourdomain.com={ hosts=<host1[:port]>[, <host2[:port]>...], keyfile=/path/to/keyfile, truststore=/path/to/truststore.pem, key_cache_millis=<cache ms>, timeout=<timeout ms> }:...
+
+The KMIP connection management only supports failover, so all requests will go through a 
+single KMIP server. There is no load balancing, as no KMIP servers (at the time of this writing)
+support read replication, or other strategies for availability.
+
+Hosts are tried in the order they appear here. Add them in the same sequence they'll fail over in.
+
+KMIP requests will fail over/retry 'max_command_retries' times (default 3)
+
+)foo")
+		  , kms_hosts(this, "kms_hosts", value_status::Used, { },
+                                R"foo(KMS host(s). 
+
+The unique name of kms host that can be referenced in table schema.
+
+host.yourdomain.com={ endpoint=<http(s)://host[:port]>, aws_access_key_id=<AWS access id>, aws_secret_access_key=<AWS secret key>, aws_profile<profile>, aws_region=<AWS region>, aws_use_ec2_credentials<bool>, aws_use_ec2_region=<bool>, aws_assume_role_arn=<AWS role arn>, master_key=<alias or id>, keyfile=/path/to/keyfile, truststore=/path/to/truststore.pem, key_cache_millis=<cache ms>, timeout=<timeout ms> }:...
+
+Actual connection can be either an explicit endpoint (<host>:<port>), or selected automatic via aws_region.
+
+If aws_use_ec2_region is true, regions is instead queried from EC2 metadata.
+
+Authentication can be explicit with aws_access_key_id and aws_secret_access_key. Either secret or both can be ommitted
+in which case the provider will try to read them from AWS credentials in ~/.aws/credentials
+
+If aws_use_ec2_credentials is true, authentication is instead queried from EC2 metadata.
+
+If aws_assume_role_arn is set, scylla will issue an AssumeRole command and use the resulting security token for key operations.
+
+master_key is an AWS KMS key id or alias from which all keys used for actual encryption of scylla data will be derived.
+This key must be pre-created with access policy allowing the above AWS id Encrypt, Decrypt and GenerateDataKey operations.
+
+)foo")
+		  , gcp_hosts(this, "gcp_hosts", value_status::Used, { },
+                                R"foo(Google Compute Engine KMS host(s). 
+
+The unique name of GCP kms host that can be referenced in table schema.
+
+gcp_project_id=<GCP project>, gcp_location=<project location>, master_key=<existing key to use for encrypting data keys>, gcp_credentials_file=<credentials json file>, gcp_impersonate_service_account=<impersonate this account for all KMS operations>,keyfile=/path/to/keyfile, truststore=/path/to/truststore.pem, key_cache_millis=<cache ms>, timeout=<timeout ms> }:...
+
+Authentication can be explicit with auth_file or by resolving default credentials (see google docs). 
+
+If use_gcp_machine_credentials is true, authentication is instead queried from GCP metadata.
+
+auth_file can contain either a user, service or impersonated service account.
+
+master_key is an GCP KMS key name from which all keys used for actual encryption of scylla data will be derived.
+This key must be pre-created with access policy allowing the above credentials Encrypt and Decrypt operations.
+
+)foo")
+        , user_info_encryption(this, "user_info_encryption", value_status::Used,
+                                { { "enabled", "false" }, { CIPHER_ALGORITHM,
+                                                "AES/CBC/PKCS5Padding" }, {
+                                                SECRET_KEY_STRENGTH, "128" },
+                                                },
+                                R"foo(Global user table encryption settings. If enabled, all user tables
+                                will be encrypted using the provided settings, unless overridden
+                                by table scylla_encryption_options.)foo")
+        , allow_per_table_encryption(this, "allow_per_table_encryption", value_status::Used, true,
+            "If 'user_info_encryption` is enabled this controls whether specifying per-table encryption using create/alter table is allowed"
+        )
+
+
+// END entry definitions
+{}
+
+static class : public configurable {
+    std::unordered_map<const db::config*, std::unique_ptr<encryption::encryption_config>> _cfgs;
+
+public:
+    void append_options(db::config& cfg, boost::program_options::options_description_easy_init& init) override {
+        // While it is fine for normal execution to have just one, static (us) encryption config, 
+        // it does not work well with unit testing, where we repeatedly create new cql_test_envs etc,
+        // since new config values will not be overwritten due to the actual named_values being shared here.
+        // Fix this (temporarily) by simply keeping a local map cfg->ecfg and using these.
+        // TODO: improve this by allowing db::config to hold named sub->configs (mapping config file objects).
+        if (_cfgs.count(&cfg)) {
+            throw std::runtime_error("Config already processed");
+        }
+        auto& ccfg = _cfgs.emplace(&cfg, std::make_unique<encryption::encryption_config>()).first->second;
+        // hook into main scylla.yaml.
+        cfg.add(ccfg->values());
+    }
+    future<notify_func> initialize_ex(const boost::program_options::variables_map& opts, const db::config& cfg, db::extensions& exts, const service_set& services) override {
+        auto ccfg = _cfgs.count(&cfg) ? std::move(_cfgs.at(&cfg)) : std::make_unique<encryption::encryption_config>();
+        _cfgs.erase(&cfg);
+        auto ctxt = co_await encryption::register_extensions(cfg, std::move(ccfg), exts, services);
+        co_return [ctxt](system_state e) -> future<> {
+            switch (e) {
+                case system_state::started:
+                    co_await ctxt->start();
+                    break;
+                case system_state::stopped:
+                    co_await ctxt->stop();
+                    break;
+                default:
+                    break;
+            }
+        };
+    }
+} cfg;

ent/encryption/encryption_config.hh

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2018 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once
+
+#include "../../utils/config_file.hh"
+
+namespace encryption {
+
+class encryption_config : public utils::config_file {
+public:
+    encryption_config();
+
+    typedef std::unordered_map<sstring, string_map> string_string_map;
+
+    named_value<sstring> system_key_directory;
+    named_value<bool> config_encryption_active;
+    named_value<sstring> config_encryption_key_name;
+    named_value<string_map> system_info_encryption;
+    named_value<string_string_map> kmip_hosts;
+    named_value<string_string_map> kms_hosts;
+    named_value<string_string_map> gcp_hosts;
+    named_value<string_map> user_info_encryption;
+    named_value<bool> allow_per_table_encryption;
+};
+
+}

ent/encryption/encryption_exceptions.hh

@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2024 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once
+
+#include "db/extensions.hh"
+
+namespace encryption {
+
+using base_error = db::extension_storage_exception;
+
+class permission_error : public db::extension_storage_permission_error {
+public:
+    using mybase = db::extension_storage_permission_error;
+    using mybase::mybase;
+};
+
+class configuration_error : public db::extension_storage_misconfigured {
+public:
+    using mybase = db::extension_storage_misconfigured;
+    using mybase::mybase;
+};
+
+class service_error : public base_error {
+public:
+    using base_error::base_error;
+};
+
+class missing_resource_error : public db::extension_storage_resource_unavailable {
+public:
+    using mybase = db::extension_storage_resource_unavailable;
+    using mybase::mybase;
+};
+
+// #4970 - not 100% correct, but network errors are 
+// generally intermittent/recoverable. Mark as a non-isolating
+// error.
+class network_error : public missing_resource_error {
+public:
+    using missing_resource_error::missing_resource_error;
+};
+
+class malformed_response_error : public service_error {
+public:
+    using service_error::service_error;
+};
+
+}
+

ent/encryption/gcp_host.hh

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2024 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once 
+
+#include <vector>
+#include <optional>
+#include <chrono>
+#include <iosfwd>
+#include <string>
+
+#include <seastar/core/future.hh>
+#include <seastar/core/shared_ptr.hh>
+
+#include "symmetric_key.hh"
+
+namespace encryption {
+
+class encryption_context;
+struct key_info;
+
+class gcp_host {
+public:
+    class impl;
+
+    template<typename T>
+    struct t_credentials_source {
+        // Path to credentials JSON file (exported from gcloud console)
+        T gcp_credentials_file;
+        // Optional service account (email address) to impersonate
+        T gcp_impersonate_service_account;
+    };
+
+    using credentials_source = t_credentials_source<std::string>;
+
+    struct host_options : public credentials_source {
+        std::string gcp_project_id;
+        std::string gcp_location;
+
+        // GCP KMS Key to encrypt data keys with. Format: <keychain>/<keyname>
+        std::string master_key;
+
+        // tls. if unspeced, use system for https
+        // GCP does not (afaik?) allow certificate auth
+        // but we keep the option available just in case.
+        std::string certfile;
+        std::string keyfile;
+        std::string truststore;
+        std::string priority_string;
+
+        std::optional<std::chrono::milliseconds> key_cache_expiry;
+        std::optional<std::chrono::milliseconds> key_cache_refresh;
+    };
+
+    using id_type = bytes;
+
+    gcp_host(encryption_context&, const std::string& name, const host_options&);
+    gcp_host(encryption_context&, const std::string& name, const std::unordered_map<sstring, sstring>&);
+    ~gcp_host();
+
+    future<> init();
+    const host_options& options() const;
+
+    struct option_override : public t_credentials_source<std::optional<std::string>> {
+        std::optional<std::string> master_key;
+    };
+
+    future<std::tuple<shared_ptr<symmetric_key>, id_type>> get_or_create_key(const key_info&, const option_override* = nullptr);
+    future<shared_ptr<symmetric_key>> get_key_by_id(const id_type&, const key_info&, const option_override* = nullptr);
+private:
+    std::unique_ptr<impl> _impl;
+};
+
+}

ent/encryption/gcp_key_provider.cc

@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2024 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#include <boost/lexical_cast.hpp>
+#include <regex>
+
+#include "gcp_key_provider.hh"
+#include "gcp_host.hh"
+
+namespace encryption {
+
+class gcp_key_provider : public key_provider {
+public:
+    gcp_key_provider(::shared_ptr<gcp_host> gcp_host, std::string name, gcp_host::option_override oov)
+        : _gcp_host(std::move(gcp_host))
+        , _name(std::move(name))
+        , _oov(std::move(oov))
+    {}
+    future<std::tuple<key_ptr, opt_bytes>> key(const key_info& info, opt_bytes id) override {
+        if (id) {
+            return _gcp_host->get_key_by_id(*id, info, &_oov).then([id](key_ptr k) {
+                return make_ready_future<std::tuple<key_ptr, opt_bytes>>(std::tuple(k, id));
+            });
+        }
+        return _gcp_host->get_or_create_key(info, &_oov).then([](std::tuple<key_ptr, opt_bytes> k_id) {
+            return make_ready_future<std::tuple<key_ptr, opt_bytes>>(k_id);
+        });
+    }
+    void print(std::ostream& os) const override {
+        os << _name;
+    }
+private:
+    ::shared_ptr<gcp_host> _gcp_host;
+    std::string _name;
+    gcp_host::option_override _oov;
+};
+
+shared_ptr<key_provider> gcp_key_provider_factory::get_provider(encryption_context& ctxt, const options& map) {
+    opt_wrapper opts(map);
+    auto gcp_host = opts("gcp_host");
+
+
+    gcp_host::option_override oov {
+        .master_key =  opts("master_key"),
+    };
+
+    oov.gcp_credentials_file = opts("gcp_credentials_file");
+    oov.gcp_impersonate_service_account = opts("gcp_impersonate_service_account");
+
+    if (!gcp_host) {
+        throw std::invalid_argument("gcp_host must be provided");
+    }
+
+    auto host = ctxt.get_gcp_host(*gcp_host);
+    auto id = gcp_host.value()
+        + ":" + oov.master_key.value_or(host->options().master_key)
+        + ":" + oov.gcp_credentials_file.value_or(host->options().gcp_credentials_file)
+        + ":" + oov.gcp_impersonate_service_account.value_or(host->options().gcp_impersonate_service_account)
+        ;
+
+    auto provider = ctxt.get_cached_provider(id);
+
+    if (!provider) {
+        provider = ::make_shared<gcp_key_provider>(host, *gcp_host, std::move(oov));
+        ctxt.cache_provider(id, provider);
+    }
+
+    return provider;
+}
+
+}

ent/encryption/gcp_key_provider.hh

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2024 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once
+
+#include "encryption.hh"
+#include "system_key.hh"
+
+namespace encryption {
+
+class gcp_key_provider_factory : public key_provider_factory {
+public:
+    shared_ptr<key_provider> get_provider(encryption_context&, const options&) override;
+};
+
+/**
+ * See comment for AWS KMS regarding system key support.
+ */
+}

ent/encryption/kmip_host.hh

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2018 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once
+
+#include <vector>
+#include <optional>
+#include <chrono>
+#include <iosfwd>
+
+#include <fmt/core.h>
+#include <fmt/ostream.h>
+
+#include <seastar/core/future.hh>
+#include <seastar/core/sstring.hh>
+#include <seastar/core/shared_ptr.hh>
+
+#include "../../bytes.hh"
+
+#include "symmetric_key.hh"
+
+namespace encryption {
+
+class symmetric_key;
+class encryption_context;
+struct key_info;
+
+class kmip_host {
+public:
+    struct host_options {
+        std::vector<sstring> hosts;
+
+        sstring username;
+        sstring password;
+
+        sstring certfile;
+        sstring keyfile;
+        sstring truststore;
+        sstring priority_string;
+
+        std::optional<std::chrono::milliseconds> key_cache_expiry;
+        std::optional<std::chrono::milliseconds> key_cache_refresh;
+
+        std::optional<size_t> max_pooled_connections_per_host;
+        std::optional<size_t> max_command_retries;
+    };
+    struct key_options {
+        sstring template_name;
+        sstring key_namespace;
+    };
+    using id_type = bytes;
+
+    kmip_host(encryption_context&, const sstring& name, const host_options&);
+    kmip_host(encryption_context&, const sstring& name, const std::unordered_map<sstring, sstring>&);
+    ~kmip_host();
+
+    future<> connect();
+    future<> disconnect();
+    future<std::tuple<shared_ptr<symmetric_key>, id_type>> get_or_create_key(const key_info&, const key_options& = {});
+    future<shared_ptr<symmetric_key>> get_key_by_id(const id_type&, std::optional<key_info> = std::nullopt);
+
+    /** for system key(s) */
+    future<shared_ptr<symmetric_key>> get_key_by_name(const sstring&);
+
+private:
+    class impl;
+    std::unique_ptr<impl> _impl;
+};
+
+std::ostream& operator<<(std::ostream&, const kmip_host::key_options&);
+
+}
+
+template <> struct fmt::formatter<encryption::kmip_host::key_options> : fmt::ostream_formatter {};

ent/encryption/kmip_key_provider.cc

@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) 2018 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#include <boost/lexical_cast.hpp>
+#include <regex>
+
+#include "utils/UUID.hh"
+#include "utils/UUID_gen.hh"
+
+#include "kmip_key_provider.hh"
+#include "kmip_host.hh"
+
+namespace encryption {
+
+class kmip_key_provider : public key_provider {
+public:
+    kmip_key_provider(::shared_ptr<kmip_host> kmip_host, kmip_host::key_options kopts, sstring name)
+        : _kmip_host(std::move(kmip_host))
+        , _kopts(std::move(kopts))
+        , _name(std::move(name))
+    {}
+    future<std::tuple<key_ptr, opt_bytes>> key(const key_info& info, opt_bytes id) override {
+        if (id) {
+            return _kmip_host->get_key_by_id(*id, info).then([id](key_ptr k) {
+                return make_ready_future<std::tuple<key_ptr, opt_bytes>>(std::tuple(k, id));
+            });
+        }
+        return _kmip_host->get_or_create_key(info, _kopts).then([](std::tuple<key_ptr, opt_bytes> k_id) {
+            return make_ready_future<std::tuple<key_ptr, opt_bytes>>(k_id);
+        });
+    }
+    void print(std::ostream& os) const override {
+        os << _name;
+        if (!_kopts.key_namespace.empty()) {
+            os << ", namespace=" << _kopts.key_namespace;
+        }
+        if (!_kopts.template_name.empty()) {
+            os << ", template=" << _kopts.template_name;
+        }
+    }
+
+private:
+    ::shared_ptr<kmip_host> _kmip_host;
+    kmip_host::key_options _kopts;
+    sstring _name;
+};
+
+
+shared_ptr<key_provider> kmip_key_provider_factory::get_provider(encryption_context& ctxt, const options& map) {
+    opt_wrapper opts(map);
+    auto host = opts(HOST_NAME);
+    if (!host) {
+        throw std::invalid_argument("kmip_host must be provided");
+    }
+    kmip_host::key_options kopts = {
+                    opts(TEMPLATE_NAME).value_or(""),
+                    opts(KEY_NAMESPACE).value_or(""),
+    };
+
+    auto cache_key = *host + ":" + boost::lexical_cast<std::string>(kopts);
+    auto provider = ctxt.get_cached_provider(cache_key);
+
+    if (!provider) {
+        provider = ::make_shared<kmip_key_provider>(ctxt.get_kmip_host(*host), std::move(kopts), *host);
+        ctxt.cache_provider(cache_key, provider);
+    }
+
+    return provider;
+}
+
+static std::optional<std::pair<sstring, sstring>> parse_kmip_host_and_path(const sstring & s) {
+    static const std::regex kmip_ex("kmip://([^/]+)/([\\w/]+)");
+
+    std::match_results<typename sstring::const_iterator> m;
+    if (std::regex_match(s.begin(), s.end(), m, kmip_ex)) {
+        return std::make_pair(sstring(m[1]), sstring(m[2]));
+    }
+    return std::nullopt;
+}
+
+kmip_system_key::kmip_system_key(encryption_context& ctxt, const sstring& s) {
+    auto p = parse_kmip_host_and_path(s);
+    if (!p) {
+        throw std::invalid_argument("Not a kmip path: " + s);
+    }
+
+    _host = ctxt.get_kmip_host(p->first);
+    _name = p->second;
+}
+
+kmip_system_key::~kmip_system_key() = default;
+
+bool kmip_system_key::is_kmip_path(const sstring& s) {
+    return parse_kmip_host_and_path(s) != std::nullopt;
+}
+
+future<shared_ptr<symmetric_key>> kmip_system_key::get_key() {
+    if (_key) {
+        return make_ready_future<shared_ptr<symmetric_key>>(_key);
+    }
+    return _host->get_key_by_name(_name).then([this](shared_ptr<symmetric_key> k) {
+        _key = k;
+        return k;
+    });
+}
+
+const sstring& kmip_system_key::name() const {
+    return _name;
+}
+
+
+}
+

ent/encryption/kmip_key_provider.hh

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2018 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+#pragma once
+
+#include "encryption.hh"
+#include "system_key.hh"
+
+namespace encryption {
+
+class kmip_key_provider_factory : public key_provider_factory {
+public:
+    shared_ptr<key_provider> get_provider(encryption_context&, const options&) override;
+};
+
+class kmip_host;
+
+class kmip_system_key : public system_key {
+    shared_ptr<symmetric_key> _key;
+    shared_ptr<kmip_host> _host;
+    sstring _name;
+public:
+    kmip_system_key(encryption_context&, const sstring&);
+    ~kmip_system_key();
+
+    static bool is_kmip_path(const sstring&);
+
+    future<shared_ptr<symmetric_key>> get_key() override;
+    const sstring& name() const override;
+    bool is_local() const override {
+        return false;
+    }
+};
+
+}

ent/encryption/kms_host.hh

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2022 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once 
+
+#include <vector>
+#include <optional>
+#include <chrono>
+#include <iosfwd>
+#include <string>
+
+#include <seastar/core/future.hh>
+#include <seastar/core/shared_ptr.hh>
+
+#include "symmetric_key.hh"
+
+namespace encryption {
+
+class encryption_context;
+struct key_info;
+
+class kms_host {
+public:
+    struct host_options {
+        std::string endpoint;
+        // or...
+        std::string host;
+        uint16_t    port;
+        bool        https = true;
+        // auth
+        std::string aws_access_key_id;
+        std::string aws_secret_access_key;
+        std::string aws_region;
+        std::string aws_profile;
+        std::string aws_assume_role_arn;
+
+        bool        aws_use_ec2_credentials;
+        bool        aws_use_ec2_region;
+
+        // key to use for keys
+        std::string master_key;
+        // tls. if unspeced, use system for https
+        // AWS does not (afaik?) allow certificate auth
+        // but we keep the option available just in case.
+        std::string certfile;
+        std::string keyfile;
+        std::string truststore;
+        std::string priority_string;
+
+        std::optional<std::chrono::milliseconds> key_cache_expiry;
+        std::optional<std::chrono::milliseconds> key_cache_refresh;
+    };
+    using id_type = bytes;
+
+    kms_host(encryption_context&, const std::string& name, const host_options&);
+    kms_host(encryption_context&, const std::string& name, const std::unordered_map<sstring, sstring>&);
+    ~kms_host();
+
+    future<> init();
+    const host_options& options() const;
+
+    struct option_override {
+        std::optional<std::string> master_key;
+        std::optional<std::string> aws_assume_role_arn;
+    };
+
+    future<std::tuple<shared_ptr<symmetric_key>, id_type>> get_or_create_key(const key_info&, const option_override* = nullptr);
+    future<shared_ptr<symmetric_key>> get_key_by_id(const id_type&, const key_info&, const option_override* = nullptr);
+private:
+    class impl;
+    std::unique_ptr<impl> _impl;
+};
+
+}

ent/encryption/kms_key_provider.cc

@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2022 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#include <boost/lexical_cast.hpp>
+#include <regex>
+
+#include "kms_key_provider.hh"
+#include "kms_host.hh"
+
+namespace encryption {
+
+class kms_key_provider : public key_provider {
+public:
+    kms_key_provider(::shared_ptr<kms_host> kms_host, std::string name, kms_host::option_override oov)
+        : _kms_host(std::move(kms_host))
+        , _name(std::move(name))
+        , _oov(std::move(oov))
+    {}
+    future<std::tuple<key_ptr, opt_bytes>> key(const key_info& info, opt_bytes id) override {
+        if (id) {
+            return _kms_host->get_key_by_id(*id, info, &_oov).then([id](key_ptr k) {
+                return make_ready_future<std::tuple<key_ptr, opt_bytes>>(std::tuple(k, id));
+            });
+        }
+        return _kms_host->get_or_create_key(info, &_oov).then([](std::tuple<key_ptr, opt_bytes> k_id) {
+            return make_ready_future<std::tuple<key_ptr, opt_bytes>>(k_id);
+        });
+    }
+    void print(std::ostream& os) const override {
+        os << _name;
+    }
+private:
+    ::shared_ptr<kms_host> _kms_host;
+    std::string _name;
+    kms_host::option_override _oov;
+};
+
+shared_ptr<key_provider> kms_key_provider_factory::get_provider(encryption_context& ctxt, const options& map) {
+    opt_wrapper opts(map);
+    auto kms_host = opts("kms_host");
+    kms_host::option_override oov {
+        .master_key = opts("master_key"),
+        .aws_assume_role_arn = opts("aws_assume_role_arn"),
+    };
+
+    if (!kms_host) {
+        throw std::invalid_argument("kms_host must be provided");
+    }
+
+    auto host = ctxt.get_kms_host(*kms_host);
+    auto id = kms_host.value() 
+        + ":" + oov.master_key.value_or(host->options().master_key)
+        + ":" + oov.aws_assume_role_arn.value_or(host->options().aws_assume_role_arn)
+        ;
+    auto provider = ctxt.get_cached_provider(id);
+
+    if (!provider) {
+        provider = ::make_shared<kms_key_provider>(host, *kms_host, std::move(oov));
+        ctxt.cache_provider(id, provider);
+    }
+
+    return provider;
+}
+
+}

ent/encryption/kms_key_provider.hh

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2022 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once
+
+#include "encryption.hh"
+#include "system_key.hh"
+
+namespace encryption {
+
+class kms_key_provider_factory : public key_provider_factory {
+public:
+    shared_ptr<key_provider> get_provider(encryption_context&, const options&) override;
+};
+
+/**
+ * As it stands today, given system_key api (gives keys), and
+ * what it is used for (config encryption), we cannot provide
+ * a KMS system key. This is because:
+ * 
+ * a.) KMS does not allow us to store a named object (key) in a secure(ish) way.
+ *     We can encrypt/decrypt and create one-off keys for local usage, which are
+ *     encoded in their own ID (see kms_host), but having a unique key from
+ *     a "path" is not possible. Esp. due to key rotation, encrypted data preamble
+ *     etc. We could keep the encrypted key material in a local file, then decrypt
+ *     it using a named key on startup, but given b.) it is dubious if this is useful.
+ * b.) System keys are only used for config encryption. The authentication config for
+ *     AWS/KMS access is typically one of the things that should be encrypted. Thus 
+ *     we would create a big chicken and egg problem here.
+ */
+}

ent/encryption/local_file_provider.cc

@@ -0,0 +1,292 @@
+/*
+ * Copyright (C) 2018 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+#include <unordered_map>
+#include <stdexcept>
+#include <regex>
+
+#include <boost/filesystem.hpp>
+
+#include <openssl/evp.h>
+#include <openssl/rand.h>
+
+#include <seastar/core/semaphore.hh>
+#include <seastar/core/seastar.hh>
+#include <seastar/core/fstream.hh>
+#include <seastar/core/reactor.hh>
+
+#include "local_file_provider.hh"
+#include "symmetric_key.hh"
+#include "encryption.hh"
+#include "encryption_exceptions.hh"
+#include "encryption_config.hh"
+#include "db/config.hh"
+
+namespace encryption {
+
+namespace bfs = std::filesystem;
+
+const sstring default_key_file_path = (bfs::path(db::config::get_conf_dir()) / "data_encryption_keys").string();
+
+static const key_info system_key_info{ "System", 0 };
+
+class local_file_provider : public key_provider {
+public:
+    local_file_provider(encryption_context& ctxt, const bfs::path& path, bool must_exist = false)
+        : local_file_provider(ctxt, sstring(bfs::absolute(path).string()), must_exist)
+    {}
+    local_file_provider(encryption_context& ctxt, const sstring& path, bool must_exist = false)
+        : _ctxt(ctxt)
+        , _path(path)
+        , _sem(1)
+        , _must_exist(must_exist)
+    {}
+    future<std::tuple<key_ptr, opt_bytes>> key(const key_info& info, opt_bytes = {}) override {
+        // TODO: assert options -> my key
+        auto i = _keys.find(info);
+        if (i != _keys.end()) {
+            return make_ready_future<std::tuple<key_ptr, opt_bytes>>(std::tuple(i->second, std::nullopt));
+        }
+        return load_or_create(info).then([](key_ptr k) {
+            return make_ready_future<std::tuple<key_ptr, opt_bytes>>(std::tuple(k, std::nullopt));
+        });
+    }
+    future<> validate() const override {
+        auto f = make_ready_future<>();
+        if (!_must_exist) {
+            return f;
+        }
+        // if we must exist, we don't change. Ok to open from all shards.
+        return f.then([this] {
+            return open_file_dma(_path, open_flags::ro).then([](file f) {
+                return f.close();
+            });
+        }).handle_exception([this](auto ep) {
+            try {
+                std::rethrow_exception(ep);
+            } catch (...) {
+                std::throw_with_nested(missing_resource_error("Could not read '" + _path + "'"));
+            }
+        });
+    }
+
+    const sstring& path() const {
+        return _path;
+    }
+    void print(std::ostream& os) const override {
+        os << "key=" << _path;
+    }
+
+private:
+    future<key_ptr> load_or_create(const key_info&);
+    future<key_ptr> load_or_create_local(const key_info&);
+    future<> read_key_file();
+    future<key_ptr> write_key_file(key_info);
+
+    std::unordered_map<key_info, key_ptr, key_info_hash> _keys;
+    encryption_context& _ctxt;
+    sstring _path;
+    semaphore _sem;
+    bool _read_file = false;
+    bool _must_exist = false;
+};
+
+shared_ptr<key_provider> local_file_provider_factory::find(encryption_context& ctxt, const sstring& path) {
+    auto p = ctxt.get_cached_provider(path);
+    if (!p) {
+        p = make_shared<local_file_provider>(ctxt, path);
+        ctxt.cache_provider(path, p);
+    }
+    return p;
+}
+
+shared_ptr<key_provider> local_file_provider_factory::get_provider(encryption_context& ctxt, const options& map) {
+    opt_wrapper opts(map);
+    return find(ctxt, opts(SECRET_KEY_FILE).value_or(default_key_file_path));
+}
+
+future<key_ptr>
+local_file_provider::load_or_create(const key_info& info) {
+    // if someone uses a system key as a table key, we could still race
+    // here. but that is a user error, so ignore
+    if (this_shard_id() == 0 || &info == &system_key_info) {
+        return load_or_create_local(info);
+    }
+
+    struct data {
+        bytes key;
+        key_info info;
+    };
+
+    /**
+     * Key files are singular. Not sharded. This would be ok if we only read from them.
+     * But in keeping with dse compat, we don't. So rather than dealing with lock files
+     * or whatnot, we simply say that a single file is handled by a single key object,
+     * and only on shard 0. So if we are not shard 0, we call to there, find our
+     * counterpart object (local_file_provider_factory::find), and as him about the
+     * key data instead. He in turn will sync on his semaphore.
+     *
+     * The downside is that we are not resilient against multiple processes messing
+     * with the key file, but neither is dse
+     */
+    return do_with(data{bytes(bytes::initialized_later(), info.len/8), info}, [this](data& i) {
+        return smp::submit_to(0, [this, &i]{
+            auto kp = static_pointer_cast<local_file_provider>(local_file_provider_factory::find(_ctxt, _path));
+            auto f = kp->load_or_create_local(i.info);
+            return f.then([&i, kp](key_ptr k) {
+                auto& kd = k->key();
+                i.key.resize(kd.size());
+                std::copy(kd.begin(), kd.end(), i.key.begin());
+            });
+        }).then([this, &i] {
+            auto k = make_shared<symmetric_key>(i.info, i.key);
+            _keys.emplace(i.info, k);
+            return make_ready_future<key_ptr>(std::move(k));
+        });
+    });
+}
+
+future<key_ptr>
+local_file_provider::load_or_create_local(const key_info& info) {
+    if (_keys.count(info)) {
+        return make_ready_future<key_ptr>(_keys.at(info));
+    }
+    return read_key_file().then([this, info] {
+        if (_keys.count(info)) {
+            return make_ready_future<key_ptr>(_keys.at(info));
+        }
+        if (info == system_key_info) {
+            if (_keys.size() != 1) {
+                _keys.clear();
+                return make_exception_future<key_ptr>(std::invalid_argument("System key must contain exactly one entry"));
+            }
+            auto k = _keys.begin()->second;
+            _keys.clear();
+            _keys.emplace(info, k);
+            return make_ready_future<key_ptr>(k);
+        }
+        // create it.
+        return write_key_file(info);
+    });
+}
+
+future<> local_file_provider::read_key_file() {
+    if (_read_file) {
+        return make_ready_future();
+    }
+
+    // #1923 - a key can have a descriptor string line "AES:128:<data>" iff user relies on
+    // defaults. Must match this as well.
+    static const std::regex key_line_expr(R"foo((\w+(?:\/\w+)?(?:\/\w+)?)\:(\d+)\:(\S+)\s*)foo");
+
+    return with_semaphore(_sem, 1, [this] {
+        // could do this twice, but it is only reading
+        return read_text_file_fully(_path).then([this](temporary_buffer<char> buf) {
+            auto i = std::cregex_iterator(buf.begin(), buf.end(), key_line_expr);
+            auto e = std::cregex_iterator();
+
+            while (i != e) {
+                std::cmatch m = *i;
+                auto alg = m[1].str();
+                auto len = std::stoul(m[2].str());
+                auto key = m[3].str();
+
+                auto info = key_info{alg, unsigned(len)};
+                if (!_keys.count(info)) {
+                    auto kb = base64_decode(key);
+                    auto k = make_shared<symmetric_key>(info, kb);
+                    _keys.emplace(info, std::move(k));
+                }
+                ++i;
+            }
+            _read_file = true;
+        }).handle_exception([this](auto ep) {
+            try {
+                std::rethrow_exception(ep);
+            } catch (std::system_error& e) {
+                if (e.code() == std::error_code(ENOENT, std::system_category())) {
+                    if (!_must_exist) {
+                        return;
+                    }
+                    std::throw_with_nested(configuration_error("Key file '" + _path + "' does not exist"));
+                }
+                std::throw_with_nested(service_error("read_key_file"));
+            } catch (std::invalid_argument& e) {
+                std::throw_with_nested(configuration_error(fmt::format("read_key_file: {}", e.what())));
+            } catch (...) {
+                std::throw_with_nested(service_error(fmt::format("read_key_file: {}", std::current_exception())));
+            }
+        });
+    });
+}
+
+future<key_ptr> local_file_provider::write_key_file(key_info info) {
+    return with_semaphore(_sem, 1, [this, info] {
+        // we can get here more than once if shards race.
+        // however, we only need to use/write the first key matching
+        // the required info.
+        if (_keys.count(info)) {
+            return make_ready_future<key_ptr>(_keys.at(info));
+        }
+
+        auto k = make_shared<symmetric_key>(info);
+
+        std::ostringstream ss;
+        for (auto& p : _keys) {
+            ss << p.first.alg << ":" << p.first.len << ":" << base64_encode(p.second->key()) << std::endl;
+        }
+        ss << info.alg << ":" << info.len << ":" << base64_encode(k->key()) << std::endl;
+        auto s = ss.str();
+        auto tmpnam = _path + ".tmp";
+        auto f = make_ready_future<>();
+        if (!_must_exist) {
+            f = seastar::recursive_touch_directory((bfs::path(tmpnam).remove_filename()).string());
+        }
+        return f.then([this, tmpnam, s] {
+            return write_text_file_fully(tmpnam, s).then([this, tmpnam] {
+                return rename_file(tmpnam, _path);
+            });
+        }).then([this, k, info] {
+            // don't cache until written
+            _keys[info] = k;
+            return make_ready_future<key_ptr>(k);
+        });
+    }).handle_exception([this](auto ep) -> key_ptr{
+        try {
+            std::rethrow_exception(ep);
+        } catch (...) {
+            std::throw_with_nested(service_error("Could not write key file '" + _path + "'"));
+        }
+    });
+}
+
+local_system_key::local_system_key(encryption_context& ctxt, const sstring& path)
+    : _provider(make_shared<local_file_provider>(ctxt, bfs::path(ctxt.config().system_key_directory()) / bfs::path(path), true))
+{}
+
+local_system_key::~local_system_key()
+{}
+
+future<shared_ptr<symmetric_key>> local_system_key::get_key() {
+    return _provider->key(system_key_info).then([](std::tuple<key_ptr, opt_bytes> k_id) {
+       return make_ready_future<shared_ptr<symmetric_key>>(std::get<0>(std::move(k_id)));
+    });
+}
+
+future<> local_system_key::validate() const {
+    // first, just validate the file provider itself
+    co_await _provider->validate();
+    // second, do an early load of the actual key to ensure file contents.
+    co_await _provider->key(system_key_info);
+}
+
+const sstring& local_system_key::name() const {
+    return _provider->path();
+}
+
+}

ent/encryption/local_file_provider.hh

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2018 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once
+
+#include "encryption.hh"
+#include "system_key.hh"
+
+namespace encryption {
+
+const extern sstring default_key_file_path;
+
+class local_file_provider;
+
+class local_file_provider_factory : public key_provider_factory {
+public:
+    static shared_ptr<key_provider> find(encryption_context&, const sstring& path);
+    shared_ptr<key_provider> get_provider(encryption_context&, const options&) override;
+};
+
+class local_system_key : public system_key {
+    shared_ptr<local_file_provider> _provider;
+public:
+    local_system_key(encryption_context&, const sstring&);
+    ~local_system_key();
+
+    future<shared_ptr<symmetric_key>> get_key() override;
+    future<> validate() const override;
+    const sstring& name() const override;
+    bool is_local() const override {
+        return true;
+    }
+};
+
+}

ent/encryption/replicated_key_provider.cc

@@ -0,0 +1,477 @@
+/*
+ * Copyright (C) 2015 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+#include <unordered_map>
+#include <stdexcept>
+#include <regex>
+#include <tuple>
+
+#include <boost/filesystem.hpp>
+
+#include <openssl/evp.h>
+#include <openssl/rand.h>
+
+#include <seastar/core/semaphore.hh>
+#include <seastar/core/seastar.hh>
+#include <seastar/core/fstream.hh>
+#include <seastar/core/reactor.hh>
+#include <seastar/core/coroutine.hh>
+
+#include <fmt/ranges.h>
+#include <fmt/std.h>
+#include "utils/to_string.hh"
+
+#include "replicated_key_provider.hh"
+#include "encryption.hh"
+#include "encryption_exceptions.hh"
+#include "local_file_provider.hh"
+#include "symmetric_key.hh"
+#include "replica/database.hh"
+#include "cql3/query_processor.hh"
+#include "cql3/untyped_result_set.hh"
+#include "utils/UUID.hh"
+#include "utils/UUID_gen.hh"
+#include "utils/hash.hh"
+#include "service/storage_service.hh"
+#include "service/migration_manager.hh"
+#include "compaction/compaction_manager.hh"
+#include "replica/distributed_loader.hh"
+#include "schema/schema_builder.hh"
+#include "db/system_keyspace.hh"
+#include "db/extensions.hh"
+#include "locator/everywhere_replication_strategy.hh"
+
+namespace encryption {
+
+static auto constexpr KSNAME = "system_replicated_keys";
+static auto constexpr TABLENAME = "encrypted_keys";
+
+static logger log("replicated_key_provider");
+
+using utils::UUID;
+
+class replicated_key_provider : public key_provider {
+public:
+    static constexpr int8_t version = 0;
+    /**
+     * Header:
+     * 1 byte version
+     * 16 bytes UUID of key
+     * 16 bytes MD5 of UUID
+     */
+    static const size_t header_size = 33;
+
+    struct key_id {
+        key_info info;
+        opt_bytes id;
+
+        key_id(key_info k, opt_bytes b = {})
+            : info(std::move(k))
+            , id(std::move(b))
+        {}
+        bool operator==(const key_id& v) const {
+            return info == v.info && id == v.id;
+        }
+    };
+
+    struct key_id_hash {
+        size_t operator()(const key_id& id) const {
+            return utils::tuple_hash()(std::tie(id.info.alg, id.info.len, id.id));
+        }
+    };
+
+    replicated_key_provider(encryption_context& ctxt, shared_ptr<system_key> system_key, shared_ptr<key_provider> local_provider)
+        : _ctxt(ctxt)
+        , _system_key(std::move(system_key))
+        , _local_provider(std::move(local_provider))
+    {}
+
+
+    future<std::tuple<key_ptr, opt_bytes>> key(const key_info&, opt_bytes = {}) override;
+    future<> validate() const override;
+    future<> maybe_initialize_tables();
+    static future<> do_initialize_tables(::replica::database& db, service::migration_manager&);
+
+    bool should_delay_read(const opt_bytes& id) const override {
+        if (!id || _initialized) {
+            return false;
+        }
+        if (!_initialized) {
+            return true;
+        }
+        auto& qp = _ctxt.get_query_processor();
+        // This check should be ok, and even somewhat redundant. "Initialized" above
+        // will only be set once we've generated/queried a key not passing through here
+        // (i.e. a key for write _or_ commit log (should we allow this)). This can only be
+        // done if:
+        // a.) Encryption was already set up, thus table existed and we waited
+        //     for distributed_tables in "ensure_populated"
+        // b.) Encryption was added. In which case we are way past bootstrap
+        //     and can receive user commands.
+        // c.) System table/commit log write, with either first use of this provider,
+        //     in which case we're creating the table (here at least) - thus fine,
+        //     or again, we've waited through "ensure_populated", so keys are
+        //     readble. At worst, we create a few extra keys.
+        // Note: currently c.) is not relevant, as we don't support system/commitlog
+        //       encryption using repl_prov.
+        return !qp.local_is_initialized();
+    }
+
+    void print(std::ostream& os) const override {
+        os << "system_key=" << _system_key->name() << ", local=" << *_local_provider;
+    }
+
+private:
+    void store_key(const key_id&, const UUID&, key_ptr);
+
+    static opt_bytes decode_id(const opt_bytes&);
+    static bytes encode_id(const UUID&);
+
+    future<std::tuple<UUID, key_ptr>> get_key(const key_info&, opt_bytes = {});
+
+    future<key_ptr> load_or_create(const key_info&);
+    future<key_ptr> load_or_create_local(const key_info&);
+    future<> read_key_file();
+    future<> write_key_file();
+
+    template<typename... Args>
+    future<::shared_ptr<cql3::untyped_result_set>> query(sstring, Args&& ...);
+
+    future<> force_blocking_flush();
+
+    encryption_context& _ctxt;
+    shared_ptr<system_key> _system_key;
+    shared_ptr<key_provider> _local_provider;
+    std::unordered_map<key_id, std::pair<UUID, key_ptr>, key_id_hash> _keys;
+
+    bool _initialized = false;
+    bool _use_cache = true;
+
+    friend class replicated_key_provider_factory;
+
+    static const utils::UUID local_fallback_uuid;
+    static const bytes local_fallback_id;
+    static const bytes_view local_fallback_bytes;
+};
+
+using namespace std::chrono_literals;
+
+static const timeout_config rkp_db_timeout_config {
+    5s, 5s, 5s, 5s, 5s, 5s, 5s,
+};
+
+static service::query_state& rkp_db_query_state() {
+    static thread_local service::client_state cs(service::client_state::internal_tag{}, rkp_db_timeout_config);
+    static thread_local service::query_state qs(cs, empty_service_permit());
+    return qs;
+}
+
+template<typename... Args>
+future<::shared_ptr<cql3::untyped_result_set>> replicated_key_provider::query(sstring q, Args&& ...params) {
+    auto mode = co_await _ctxt.get_storage_service().local().get_operation_mode();
+    if (mode != service::storage_service::mode::STARTING) {
+        co_return co_await _ctxt.get_query_processor().local().execute_internal(q, { std::forward<Args>(params)...}, cql3::query_processor::cache_internal::no);
+    }
+    co_return co_await _ctxt.get_query_processor().local().execute_internal(q, db::consistency_level::ONE, rkp_db_query_state(), { std::forward<Args>(params)...}, cql3::query_processor::cache_internal::no);
+}
+
+future<> replicated_key_provider::force_blocking_flush() {
+    return _ctxt.get_database().invoke_on_all([](replica::database& db) {
+        // if (!Boolean.getBoolean("cassandra.unsafesystem"))
+        replica::column_family& cf = db.find_column_family(KSNAME, TABLENAME);
+        return cf.flush();
+    });
+}
+
+void replicated_key_provider::store_key(const key_id& id, const UUID& uuid, key_ptr k) {
+    if (!_use_cache) {
+        return;
+    }
+    _keys[id] = std::make_pair(uuid, k);
+    if (!id.id) {
+        _keys[key_id(id.info, uuid.serialize())] = std::make_pair(uuid, k);
+    }
+}
+
+opt_bytes replicated_key_provider::decode_id(const opt_bytes& b) {
+    if (b) {
+        auto i = b->begin();
+        auto v = *i++;
+        if (v == version && b->size() == 33) {
+            bytes id(i + 1, i + 1 + 16);
+            bytes md(i + 1 + 16, b->end());
+            if (calculate_md5(id) == md) {
+                return id;
+            }
+        }
+    }
+    return std::nullopt;
+}
+
+bytes replicated_key_provider::encode_id(const UUID& uuid) {
+    bytes b{bytes::initialized_later(), header_size};
+    auto i = b.begin();
+    *i++ = version;
+    uuid.serialize(i);
+    auto md = calculate_md5(b, 1, 16);
+    std::copy(md.begin(), md.end(), i);
+    return b;
+}
+
+const utils::UUID replicated_key_provider::local_fallback_uuid(0u, 0u); // not valid!
+const bytes replicated_key_provider::local_fallback_id = encode_id(local_fallback_uuid);
+const bytes_view replicated_key_provider::local_fallback_bytes(local_fallback_id.data() + 1, 16);
+
+future<std::tuple<key_ptr, opt_bytes>> replicated_key_provider::key(const key_info& info, opt_bytes input) {
+    opt_bytes id;
+
+    if (input) { //reading header?
+        auto v = *input;
+        if (v[0] == version) {
+            bytes bid(v.begin() + 1, v.begin() + 1 + 16);
+            bytes md(v.begin() + 1 + 16, v.begin() + 1 + 32);
+            if (calculate_md5(bid) == md) {
+                id = bid;
+            }
+        }
+    }
+
+    bool try_local = id == local_fallback_bytes;
+
+    // if the id indicates the key came from local fallback, don't even
+    // try keyspace lookup.
+    if (!try_local) {
+        try {
+            auto [uuid, k] = co_await get_key(info, std::move(id));
+            co_return std::make_tuple(k, encode_id(uuid));
+        } catch (std::invalid_argument& e) {
+            std::throw_with_nested(configuration_error(e.what()));
+        } catch (...) {
+            auto ep = std::current_exception();
+            log.warn("Exception looking up key {}: {}", info, ep);
+            if (_local_provider) {
+                try {
+                    std::rethrow_exception(ep);
+                } catch (replica::no_such_keyspace&) {
+                } catch (exceptions::invalid_request_exception&) {
+                } catch (exceptions::read_failure_exception&) {
+                } catch (...) {
+                    std::throw_with_nested(service_error(fmt::format("key: {}", std::current_exception())));
+                }
+                if (!id) {
+                    try_local = true;
+                }
+            }
+            if (!try_local) {
+                std::throw_with_nested(service_error(fmt::format("key: {}", std::current_exception())));
+            }
+        }
+    }
+
+    log.warn("Falling back to local key {}", info);
+    auto [k, nid] = co_await _local_provider->key(info, id);
+    if (nid && nid != id) {
+        // local provider does not give ids.
+        throw malformed_response_error("Expected null id back from local provider");
+    }
+    co_return std::make_tuple(k, local_fallback_id);
+}
+
+future<std::tuple<UUID, key_ptr>> replicated_key_provider::get_key(const key_info& info, opt_bytes opt_id) {
+    if (!_initialized) {
+        co_await maybe_initialize_tables();
+    }
+
+    key_id id(info, std::move(opt_id));
+    auto i = _keys.find(id);
+    if (i != _keys.end()) {
+        co_return std::tuple(i->second.first, i->second.second);
+    }
+
+    // TODO: origin does non-cql acquire of all available keys from
+    // replicas in the "host_ids" table iff we get here during boot.
+    // For now, ignore this and assume that if we have a sstable with
+    // key X, we should have a local replica of X as well, given
+    // the "everywhere strategy of the keys table.
+
+    auto cipher = info.alg.substr(0, info.alg.find('/')); // e.g. "AES"
+
+    UUID uuid;
+    shared_ptr<cql3::untyped_result_set> res;
+
+    if (id.id) {
+        uuid = utils::UUID_gen::get_UUID(*id.id);
+        log.debug("Finding key {} ({})", uuid, info);
+        auto s = fmt::format("SELECT * FROM {}.{} WHERE key_file=? AND cipher=? AND strength=? AND key_id=?;", KSNAME, TABLENAME);
+        res = co_await query(std::move(s), _system_key->name(), cipher, int32_t(id.info.len), uuid);
+
+        // if we find nothing, and we actually queried a specific key (by uuid), we've failed.
+        if (res->empty()) {
+            log.debug("Could not find key {}", id.id);
+            throw std::runtime_error(fmt::format("Unable to find key for cipher={} strength={} id={}", cipher, id.info.len, uuid));
+        }
+    } else {
+        log.debug("Finding key ({})", info);
+        auto s = fmt::format("SELECT * FROM {}.{} WHERE key_file=? AND cipher=? AND strength=? LIMIT 1;", KSNAME, TABLENAME);
+        res = co_await query(std::move(s), _system_key->name(), cipher, int32_t(id.info.len));
+    }
+
+    // otoh, if we don't need a specific key, we can just create a new one (writing a sstable)
+    if (res->empty()) {
+        uuid = utils::UUID_gen::get_time_UUID();
+
+        log.debug("No key found. Generating {}", uuid);
+
+        auto k = make_shared<symmetric_key>(id.info);
+        store_key(id, uuid, k);
+
+        auto b = co_await _system_key->encrypt(k->key());
+        auto ks = base64_encode(b);
+        log.trace("Inserting generated key {}", uuid);
+        co_await query(fmt::format("INSERT INTO {}.{} (key_file, cipher, strength, key_id, key) VALUES (?, ?, ?, ?, ?)", 
+            KSNAME, TABLENAME), _system_key->name(), cipher, int32_t(id.info.len), uuid, ks
+        );
+        log.trace("Flushing key table");
+        co_await force_blocking_flush();
+
+        co_return std::tuple(uuid, k);
+    }
+
+    // found it
+    auto& row = res->one();
+    uuid = row.get_as<UUID>("key_id");
+    auto ks = row.get_as<sstring>("key");
+    auto kb = base64_decode(ks);
+    auto b = co_await _system_key->decrypt(kb);
+    auto k = make_shared<symmetric_key>(id.info, b);
+    store_key(id, uuid, k);
+
+    co_return std::tuple(uuid, k);
+}
+
+future<> replicated_key_provider::validate() const {
+    try {
+        co_await _system_key->validate();
+    } catch (...) {
+        std::throw_with_nested(std::invalid_argument(fmt::format("Could not validate system key: {}", _system_key->name())));
+    }
+    if (_local_provider){
+        co_await _local_provider->validate();
+    }
+}
+
+schema_ptr encrypted_keys_table() {
+    static thread_local auto schema = [] {
+        auto id = generate_legacy_id(KSNAME, TABLENAME);
+        return schema_builder(KSNAME, TABLENAME, std::make_optional(id))
+                .with_column("key_file", utf8_type, column_kind::partition_key)
+                .with_column("cipher", utf8_type, column_kind::partition_key)
+                .with_column("strength", int32_type, column_kind::clustering_key)
+                .with_column("key_id", timeuuid_type, column_kind::clustering_key)
+                .with_column("key", utf8_type)
+                .with_hash_version()
+                .build();
+    }();
+    return schema;
+}
+
+future<> replicated_key_provider::maybe_initialize_tables() {
+    if (!_initialized) {
+        co_await do_initialize_tables(_ctxt.get_database().local(), _ctxt.get_migration_manager().local());
+        _initialized = true;
+    }
+}
+
+future<> replicated_key_provider::do_initialize_tables(::replica::database& db, service::migration_manager& mm) {
+    if (db.has_schema(KSNAME, TABLENAME)) {
+        co_return;
+    }
+
+    log.debug("Creating keyspace and table");
+    if (!db.has_keyspace(KSNAME)) {
+        auto group0_guard = co_await mm.start_group0_operation();
+        auto ts = group0_guard.write_timestamp();
+        try {
+            auto ksm = keyspace_metadata::new_keyspace(
+                    KSNAME,
+                    "org.apache.cassandra.locator.EverywhereStrategy",
+                    {},
+                    std::nullopt,
+                    true);
+            co_await mm.announce(service::prepare_new_keyspace_announcement(db, ksm, ts), std::move(group0_guard), fmt::format("encryption at rest: create keyspace {}", KSNAME));
+        } catch (exceptions::already_exists_exception&) {
+        }
+    }
+    auto group0_guard = co_await mm.start_group0_operation();
+    auto ts = group0_guard.write_timestamp();
+    try {
+        co_await mm.announce(co_await service::prepare_new_column_family_announcement(mm.get_storage_proxy(), encrypted_keys_table(), ts), std::move(group0_guard),
+                             fmt::format("encryption at rest: create table {}.{}", KSNAME, TABLENAME));
+    } catch (exceptions::already_exists_exception&) {
+    }
+    auto& ks = db.find_keyspace(KSNAME);
+    auto& rs = ks.get_replication_strategy();
+    // should perhaps check name also..
+    if (rs.get_type() != locator::replication_strategy_type::everywhere_topology) {
+        // TODO: reset to everywhere + repair.
+    }
+}
+
+const size_t replicated_key_provider::header_size;
+
+replicated_key_provider_factory::replicated_key_provider_factory()
+{}
+
+replicated_key_provider_factory::~replicated_key_provider_factory()
+{}
+
+namespace bfs = std::filesystem;
+
+shared_ptr<key_provider> replicated_key_provider_factory::get_provider(encryption_context& ctxt, const options& map) {
+    opt_wrapper opts(map);
+    auto system_key_name = opts(SYSTEM_KEY_FILE).value_or("system_key");
+    if (system_key_name.find('/') != sstring::npos) {
+        throw std::invalid_argument("system_key cannot contain '/'");
+    }
+
+    auto system_key = ctxt.get_system_key(system_key_name);
+    auto local_key_file = bfs::absolute(bfs::path(opts(SECRET_KEY_FILE).value_or(default_key_file_path)));
+
+    if (system_key->is_local() && bfs::absolute(bfs::path(system_key->name())) == local_key_file) {
+        throw std::invalid_argument("system key and local key cannot be the same");
+    }
+
+    auto name = system_key->name() + ":" + local_key_file.string();
+    auto debug = opts("DEBUG");
+    if (debug) {
+        name = name + ":" + *debug;
+    }
+    auto p = ctxt.get_cached_provider(name);
+    if (!p) {
+        auto rp = seastar::make_shared<replicated_key_provider>(ctxt, std::move(system_key), local_file_provider_factory::find(ctxt, local_key_file.string()));
+        ctxt.cache_provider(name, rp);
+
+        if (debug && debug->find("nocache") != sstring::npos) {
+            log.debug("Turn off cache");
+            rp->_use_cache = false;
+        }
+        p = std::move(rp);
+    }
+
+    return p;
+}
+
+void replicated_key_provider_factory::init(db::extensions& exts) {
+    exts.add_extension_internal_keyspace(KSNAME);
+}
+
+future<> replicated_key_provider_factory::on_started(::replica::database& db, service::migration_manager& mm) {
+    return replicated_key_provider::do_initialize_tables(db, mm);
+}
+
+}

ent/encryption/replicated_key_provider.hh

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2015 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once
+
+#include "encryption.hh"
+
+namespace db {
+class extensions;
+}
+
+namespace replica {
+class database;
+}
+
+namespace service {
+class migration_manager;
+}
+
+namespace encryption {
+
+class replicated_key_provider_factory : public key_provider_factory {
+public:
+    replicated_key_provider_factory();
+    ~replicated_key_provider_factory();
+
+    shared_ptr<key_provider> get_provider(encryption_context&, const options&) override;
+
+    static void init(db::extensions&);
+    static future<> on_started(::replica::database&, service::migration_manager&);
+};
+
+}

ent/encryption/symmetric_key.cc

@@ -0,0 +1,396 @@
+/*
+ * Copyright (C) 2018 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+#include <stdexcept>
+#include <regex>
+#include <algorithm>
+
+#include <openssl/evp.h>
+#include <openssl/rand.h>
+#include <openssl/err.h>
+
+#if OPENSSL_VERSION_NUMBER >= (3<<28)
+#   include <openssl/provider.h>
+#endif
+
+#include <seastar/core/align.hh>
+#include <seastar/core/print.hh>
+
+#include "symmetric_key.hh"
+#include "utils/hash.hh"
+
+namespace {
+struct openssl_env {
+    OSSL_PROVIDER* legacy_provider = nullptr;
+    OSSL_PROVIDER* default_provider = nullptr;
+    openssl_env() {
+        OpenSSL_add_all_ciphers();
+#if OPENSSL_VERSION_NUMBER >= (3<<28)
+        legacy_provider = OSSL_PROVIDER_load(NULL, "legacy");
+        default_provider = OSSL_PROVIDER_load(NULL, "default");
+#endif
+    }
+    ~openssl_env() {
+        OSSL_PROVIDER_unload(legacy_provider);
+        OSSL_PROVIDER_unload(default_provider);
+    }
+};
+static const openssl_env ossl_env;
+}
+
+std::ostream& encryption::operator<<(std::ostream& os, const key_info& info) {
+    return os << info.alg << ":" << info.len;
+}
+
+static void throw_evp_error(std::string msg) {
+    auto e = ERR_get_error();
+    if (e != 0) {
+        char buf[512];
+        ERR_error_string_n(e, buf, sizeof(buf));
+        msg += "(" + std::string(buf) + ")";
+    }
+    throw std::runtime_error(msg);
+}
+
+bool encryption::key_info::compatible(const key_info& rhs) const {
+    sstring malg, halg;
+    std::tie(malg, std::ignore, std::ignore) = parse_key_spec(alg);
+    std::tie(halg, std::ignore, std::ignore) = parse_key_spec(rhs.alg);
+    if (malg != halg) {
+        return false;
+    }
+    // If lengths differ we need to actual create keys to
+    // check what the true lengths are. Since openssl and
+    // java designators count different for DES etc.
+    if (len != rhs.len) {
+        symmetric_key k1(*this);
+        symmetric_key k2(rhs);
+        if (k1.key().size() != k2.key().size()) {
+            return false;
+        }
+    }
+    return true;
+}
+
+std::tuple<sstring, sstring, sstring>
+encryption::parse_key_spec(const sstring& alg) {
+    static const std::regex alg_exp(R"foo(^(\w+)(?:\/(\w+))?(?:\/(\w+))?$)foo");
+
+    std::cmatch m;
+    if (!std::regex_match(alg.begin(), alg.end(), m, alg_exp)) {
+        throw std::invalid_argument("Invalid algorithm string: " + alg);
+    }
+
+    auto type = m[1].str();
+    auto mode = m[2].str();
+    auto padd = m[3].str();
+
+    std::transform(type.begin(), type.end(), type.begin(), ::tolower);
+    std::transform(mode.begin(), mode.end(), mode.begin(), ::tolower);
+    std::transform(padd.begin(), padd.end(), padd.begin(), ::tolower);
+
+    static const std::string padding = "padding";
+    if (padd.size() > padding.size() && std::equal(padding.rbegin(), padding.rend(), padd.rbegin())) {
+        padd.resize(padd.size() - padding.size());
+    }
+
+    return std::make_tuple<sstring, sstring, sstring>(type, mode, padd);
+}
+
+std::tuple<sstring, sstring, sstring> encryption::parse_key_spec_and_validate_defaults(const sstring& alg) {
+    auto [type, mode, padd] = parse_key_spec(alg);
+
+    // openssl AND kmip server(s?) does not allow missing block mode. so default one.
+    if (mode.empty()) {
+        mode = "cbc";
+    }
+
+    // OpenSSL only supports one form of padding. We used to just allow
+    // non-empty string -> pkcs5/pcks7. Better to verify
+    // (note: pcks5 is sortof a misnomeanor here, as in the Sun world, it
+    // sort of means "pkcs7 with automatic block size" - which is pretty
+    // much how things are in the OpenSSL universe as well)
+    if (padd == "no") {
+        padd = "";
+    }
+    if (!padd.empty() && padd != "pkcs5" && padd != "pkcs" && padd != "pkcs7") {
+        throw std::invalid_argument("non-supported padding option: " + padd);
+    }
+
+    return { type, mode, padd };
+}
+
+encryption::symmetric_key::symmetric_key(const key_info& info, const bytes& key)
+    : _ctxt(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free)
+    , _info(info)
+    , _key(key)
+{
+    if (!_ctxt) {
+        throw std::bad_alloc();
+    }
+
+    sstring type, mode, padd;
+    std::tie(type, mode, padd) = parse_key_spec_and_validate_defaults(info.alg);
+
+    // Note: we are using some types here that are explicitly marked as "unsupported - placeholder"
+    // in gnutls.
+
+    // camel case vs. dash
+    if (type == "desede") {
+        type = "des-ede";
+        // and 168-bits desede is ede3 in openssl...
+        if (info.len > 16*8) {
+            type = "des-ede3";
+        }
+    }
+
+    auto str = fmt::format("{}-{}-{}", type, info.len, mode);
+    auto cipher = EVP_get_cipherbyname(str.c_str());
+
+    if (!cipher) {
+        str = fmt::format("{}-{}", type, mode);
+        cipher = EVP_get_cipherbyname(str.c_str());
+    }
+    if (!cipher) {
+        str = fmt::format("{}-{}", type, info.len);
+        cipher = EVP_get_cipherbyname(str.c_str());
+    }
+    if (!cipher) {
+        str = type;
+        cipher = EVP_get_cipherbyname(str.c_str());
+    }
+    if (!cipher) {
+        throw_evp_error("Invalid algorithm: " + info.alg);
+    }
+
+    size_t len = EVP_CIPHER_key_length(cipher);
+
+    if ((_info.len/8) != len) {
+        if (!EVP_CipherInit_ex(*this, cipher, nullptr, nullptr, nullptr, 0)) {
+            throw_evp_error("Could not initialize cipher");
+        }
+        auto dlen = _info.len/8;
+        // Openssl describes des-56 length as 64 (counts parity),
+        // des-ede-112 as 128 etc...
+        // do some special casing...
+        if ((type == "des" || type == "des-ede" || type == "des-ede3") && (dlen & 7) != 0) {
+            dlen = align_up(dlen, 8u);
+        }
+        // if we had to find a cipher without explicit key length (like rc2),
+        // try to set the key length to the desired strength.
+        if (!EVP_CIPHER_CTX_set_key_length(*this, dlen)) {
+            throw_evp_error(fmt::format("Invalid length {} for resolved type {} (wanted {})", len*8, str, _info.len));
+        }
+
+        len = EVP_CIPHER_key_length(cipher);
+    }
+
+
+    if (_key.empty()) {
+        _key.resize(len);
+        if (!RAND_bytes(reinterpret_cast<uint8_t*>(_key.data()), _key.size())) {
+            throw_evp_error(fmt::format("Could not generate key: {}", info.alg));
+        }
+    }
+    if (_key.size() < len) {
+        throw std::invalid_argument(fmt::format("Invalid key data length {} for resolved type {} ({})", _key.size()*8, str, len*8));
+    }
+
+    if (!EVP_CipherInit_ex(*this, cipher, nullptr,
+                    reinterpret_cast<const uint8_t*>(_key.data()), nullptr,
+                    0)) {
+        throw_evp_error("Could not initialize cipher from key materiel");
+    }
+
+    _iv_len = EVP_CIPHER_CTX_iv_length(*this);
+    _block_size = EVP_CIPHER_CTX_block_size(*this);
+    _padding = !padd.empty();
+
+}
+
+std::string encryption::symmetric_key::validate_exact_info_result() const {
+    auto [types, modes, padds] = parse_key_spec(_info.alg);
+
+    auto cipher = EVP_CIPHER_CTX_get0_cipher(*this);
+    auto len = EVP_CIPHER_key_length(cipher);
+    auto mode = EVP_CIPHER_get_mode(cipher);
+
+    std::ostringstream ss;
+
+    if (unsigned(len)*8 != align_up(_info.len, 16u)) {
+        ss << "Length " << len*8 << " differs from requested " << _info.len << std::endl;
+    }
+
+    static std::unordered_map<int, std::string> openssl_modes({
+        { EVP_CIPH_ECB_MODE, "ecb"  },
+        { EVP_CIPH_CBC_MODE, "cbc"  },
+        { EVP_CIPH_CFB_MODE, "cfb"  }, 
+        { EVP_CIPH_OFB_MODE, "ofb"  },
+        { EVP_CIPH_CTR_MODE, "ctr"  },
+        { EVP_CIPH_GCM_MODE, "cgm"  },
+        { EVP_CIPH_CCM_MODE, "ccm"  },
+        { EVP_CIPH_XTS_MODE, "xts"  },
+        { EVP_CIPH_WRAP_MODE, "wrap"},
+        { EVP_CIPH_OCB_MODE, "ocb"  },
+        { EVP_CIPH_SIV_MODE, "siv"  },
+    });
+
+    auto i = openssl_modes.find(mode);
+    if (i != openssl_modes.end() && i->second != modes) {
+        ss << _info << ": " << "Block mode " << i->second << " differers from requested " << modes << std::endl;
+    }
+
+    if ((!padds.empty() && padds != "no") != _padding) {
+        ss << _info << ": " << "Padding (" << bool(_padding) << " differs from requested " << padds << std::endl;
+    }
+
+    return ss.str();
+}
+
+void encryption::symmetric_key::generate_iv_impl(uint8_t* dst, size_t s) const {
+    if (s < _iv_len) {
+        throw std::invalid_argument("Buffer underflow");
+    }
+    if (!RAND_bytes(dst, s)) {
+        throw_evp_error("Could not generate initialization vector");
+    }
+}
+
+void encryption::symmetric_key::transform_unpadded_impl(const uint8_t* input,
+                size_t input_len, uint8_t* output, const uint8_t* iv, mode m) const {
+    if (!EVP_CipherInit_ex(*this, nullptr, nullptr,
+                    reinterpret_cast<const uint8_t*>(_key.data()), iv, int(m))) {
+        throw_evp_error("Could not initialize cipher (transform)");
+    }
+    if (!EVP_CIPHER_CTX_set_padding(*this, 0)) {
+        throw_evp_error("Could not disable padding");
+    }
+
+    if (input_len & (_block_size - 1)) {
+        throw std::invalid_argument("Data must be aligned to 'blocksize'");
+    }
+
+    int outl = 0;
+    auto res = m == mode::decrypt ?
+                    EVP_DecryptUpdate(*this, output, &outl, input,
+                                    int(input_len)) :
+                    EVP_EncryptUpdate(*this, output, &outl, input,
+                                    int(input_len));
+
+    if (!res || outl != int(input_len)) {
+        throw std::runtime_error("transformation failed");
+    }
+}
+
+size_t encryption::symmetric_key::decrypt_impl(const uint8_t* input,
+                size_t input_len, uint8_t* output, size_t output_len,
+                const uint8_t* iv) const {
+    if (!EVP_CipherInit_ex(*this, nullptr, nullptr,
+                    reinterpret_cast<const uint8_t*>(_key.data()), iv, 0)) {
+        throw_evp_error("Could not initialize cipher (decrypt)");
+    }
+    if (!EVP_CIPHER_CTX_set_padding(*this, int(_padding))) {
+        throw_evp_error("Could not initialize padding");
+    }
+
+    // normal case, caller provides output enough to deal with any padding.
+    // in padding case, max out size is input_len - 1.
+    if (input_len <= output_len) {
+        // one go.
+        int outl = 0;
+        int finl = 0;
+        if (!EVP_DecryptUpdate(*this, output, &outl, input, int(input_len))) {
+            throw_evp_error("decryption failed");
+        }
+        if (!EVP_DecryptFinal(*this, output + outl, &finl)) {
+            throw_evp_error("decryption failed");
+        }
+
+        return outl + finl;
+    }
+
+    // meh. must provide block padding.
+    constexpr size_t local_buf_size = 1024;
+
+    static thread_local std::vector<unsigned char> cached_buf;
+
+    if (cached_buf.size() < local_buf_size + _block_size) [[unlikely]] {
+        cached_buf.resize(local_buf_size + _block_size);
+    }
+
+    auto buf = cached_buf.data();
+    size_t res = 0;
+    while (input_len) {
+        auto n = std::min(input_len, local_buf_size);
+        int outl = 0;
+        if (!EVP_DecryptUpdate(*this, buf, &outl, input, int(n))) {
+            throw std::runtime_error("decryption failed");
+        }
+        if (n < local_buf_size) {
+            // last block
+            int finl = 0;
+            if (!EVP_DecryptFinal(*this, buf + outl, &finl)) {
+                throw std::runtime_error("decryption failed");
+            }
+            outl += finl;
+        }
+        if ((res + outl) > output_len) {
+            throw std::invalid_argument("Output buffer too small");
+        }
+        output = std::copy(buf, buf + outl, output);
+        res += outl;
+        input_len -= n;
+        input += n;
+    }
+
+    return res;
+}
+
+size_t encryption::symmetric_key::encrypted_size(size_t n) const {
+    // encryption always adds padding. So if n is multiple of blocksize
+    // the size is n + blocksize. But if its not, things are "better"...
+    return _block_size + align_down<size_t>(n, _block_size);
+}
+
+size_t encryption::symmetric_key::encrypt_impl(const uint8_t* input,
+                size_t input_len, uint8_t* output, size_t output_len,
+                const uint8_t* iv) const {
+    if (output_len < encrypted_size(input_len)) {
+        throw std::invalid_argument("Insufficient buffer");
+    }
+
+    if (!EVP_CipherInit_ex(*this, nullptr, nullptr,
+                    reinterpret_cast<const uint8_t*>(_key.data()), iv, 1)) {
+        throw_evp_error("Could not initialize cipher (encrypt)");
+    }
+    if (!EVP_CIPHER_CTX_set_padding(*this, int(_padding))) {
+        throw_evp_error("Could not initialize padding");
+    }
+
+    int outl = 0;
+    int finl = 0;
+    if (!EVP_EncryptUpdate(*this, output, &outl, input, int(input_len))) {
+        throw_evp_error("encryption failed");
+    }
+    if (!EVP_EncryptFinal(*this, output + outl, &finl)) {
+        throw_evp_error("encryption failed");
+    }
+    return outl + finl;
+}
+
+bool encryption::operator==(const key_info& k1, const key_info& k2) {
+    return k1.alg == k2.alg && k1.len == k2.len;
+}
+
+bool encryption::operator!=(const key_info& k1, const key_info& k2) {
+    return !(k1 == k2);
+}
+
+size_t encryption::key_info_hash::operator()(const key_info& e) const {
+    return utils::tuple_hash()(std::tie(e.alg, e.len));
+}

ent/encryption/symmetric_key.hh

@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2018 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once
+
+#include <memory>
+#include <tuple>
+#include <iosfwd>
+#include <fmt/core.h>
+#include <fmt/ostream.h>
+
+#include "../../bytes.hh"
+
+// forward declare openssl evp.
+extern "C" {
+struct evp_cipher_ctx_st;
+}
+
+namespace encryption {
+
+struct key_info {
+    sstring alg;
+    unsigned len;
+
+    bool compatible(const key_info&) const;
+};
+
+bool operator==(const key_info& k1, const key_info& k2);
+bool operator!=(const key_info& k1, const key_info& k2);
+std::ostream& operator<<(std::ostream&, const key_info&);
+
+struct key_info_hash {
+    size_t operator()(const key_info& e) const;
+};
+
+std::tuple<sstring, sstring, sstring> parse_key_spec(const sstring&);
+
+// shared between key & kmip
+std::tuple<sstring, sstring, sstring> parse_key_spec_and_validate_defaults(const sstring&);
+
+class symmetric_key {
+    std::unique_ptr<evp_cipher_ctx_st, void (*)(evp_cipher_ctx_st*)> _ctxt;
+    key_info _info;
+    bytes _key;
+    unsigned _iv_len = 0;
+    unsigned _block_size = 0;
+    bool _padding = true;
+
+    operator evp_cipher_ctx_st *() const {
+        return _ctxt.get();
+    }
+
+    void generate_iv_impl(uint8_t* dst, size_t) const;
+    size_t decrypt_impl(const uint8_t* input, size_t input_len, uint8_t* output,
+                    size_t output_len, const uint8_t* iv) const;
+    size_t encrypt_impl(const uint8_t* input, size_t input_len, uint8_t* output,
+                    size_t output_len, const uint8_t* iv) const;
+
+public:
+    symmetric_key(const key_info& info, const bytes& key = { });
+
+    const key_info& info() const {
+        return _info;
+    }
+    const bytes& key() const {
+        return _key;
+    }
+    size_t iv_len() const {
+        return _iv_len;
+    }
+    size_t block_size() const {
+        return _block_size;
+    }
+
+    /**
+     * Evaluates whether or not the key info provided resulted in
+     * the exact same result from openssl, i.e. whether the combination
+     * of alg/block mode/padding etc was actually fully valid (or our 
+     * heuristics have issues)
+    */
+    std::string validate_exact_info_result() const;
+
+    /**
+     * Write a random IV to dst. Must be iv_len() sized or larger
+     */
+    template<typename T>
+    void generate_iv(T* dst, size_t s) const {
+        static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v<T>);
+        generate_iv_impl(reinterpret_cast<uint8_t *>(dst), s);
+    }
+
+    // returns minimal buffer size required to encrypt n bytes. I.e.
+    // block alignment
+    size_t encrypted_size(size_t n) const;
+
+    template<typename T, typename V, typename I = char>
+    size_t decrypt(const T* input, size_t input_len, V* output,
+                    size_t output_len, const I* iv = nullptr) const {
+        static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v<T>);
+        return decrypt_impl(reinterpret_cast<const uint8_t*>(input), input_len,
+                        reinterpret_cast<uint8_t *>(output), output_len,
+                        reinterpret_cast<const uint8_t*>(iv));
+    }
+    template<typename T, typename V, typename I = char>
+    size_t encrypt(const T* input, size_t input_len, V* output,
+                    size_t output_len, const I* iv = nullptr) const {
+        static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v<T>);
+        return encrypt_impl(reinterpret_cast<const uint8_t*>(input), input_len,
+                        reinterpret_cast<uint8_t *>(output), output_len,
+                        reinterpret_cast<const uint8_t*>(iv));
+    }
+
+    enum class mode {
+        decrypt, encrypt,
+    };
+    template<typename T, typename V, typename I = char>
+    void transform_unpadded(mode m, const T* input, size_t input_len, V* output,
+                    const I* iv = nullptr) const {
+        static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v<T>);
+        return transform_unpadded_impl(reinterpret_cast<const uint8_t*>(input),
+                        input_len, reinterpret_cast<uint8_t *>(output),
+                        reinterpret_cast<const uint8_t*>(iv), m);
+    }
+    template<typename T, typename V, typename I = char>
+    void encrypt_unpadded(const T* input, size_t input_len, V* output,
+                    const I* iv = nullptr) const {
+        static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v<T>);
+        return transform_unpadded_impl(reinterpret_cast<const uint8_t*>(input),
+                        input_len, reinterpret_cast<uint8_t *>(output),
+                        reinterpret_cast<const uint8_t*>(iv), mode::encrypt);
+    }
+    template<typename T, typename V, typename I = char>
+    void decrypt_unpadded(const T* input, size_t input_len, V* output,
+                    const I* iv = nullptr) const {
+        static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v<T>);
+        return transform_unpadded_impl(reinterpret_cast<const uint8_t*>(input),
+                        input_len, reinterpret_cast<uint8_t *>(output),
+                        reinterpret_cast<const uint8_t*>(iv), mode::decrypt);
+    }
+
+private:
+    void transform_unpadded_impl(const uint8_t* input, size_t input_len,
+                    uint8_t* output, const uint8_t* iv, mode) const;
+};
+
+}
+
+template <> struct fmt::formatter<encryption::key_info> : fmt::ostream_formatter {};

ent/encryption/system_key.cc

@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2015 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+#include <stdexcept>
+#include <regex>
+#include <algorithm>
+#include <unordered_map>
+
+#include <openssl/evp.h>
+#include <openssl/rand.h>
+
+#include <seastar/core/align.hh>
+
+#include "symmetric_key.hh"
+#include "system_key.hh"
+
+future<> encryption::system_key::validate() const {
+    return make_ready_future<>();
+}
+
+future<sstring> encryption::system_key::decrypt(const sstring& s) {
+    auto b = base64_decode(s);
+    return decrypt(b).then([](bytes b) {
+        return make_ready_future<sstring>(sstring(b.begin(), b.end()));
+    });
+}
+
+future<sstring> encryption::system_key::encrypt(const sstring& s) {
+    return encrypt(bytes(s.begin(), s.end())).then([](bytes b) {
+        return make_ready_future<sstring>(base64_encode(b));
+    });
+}
+
+future<bytes> encryption::system_key::encrypt(const bytes& b) {
+    return get_key().then([b](shared_ptr<symmetric_key> k) {
+        auto i = k->iv_len();
+        auto n = k->encrypted_size(b.size());
+        bytes res(bytes::initialized_later(), n + i);
+        k->generate_iv(reinterpret_cast<char*>(res.data()), i);
+        n = k->encrypt(reinterpret_cast<const char*>(b.data()), b.size()
+                        , reinterpret_cast<char*>(res.data()) + i, res.size() - i
+                        , reinterpret_cast<const char*>(res.data()));
+        res.resize(n + i);
+        return make_ready_future<bytes>(std::move(res));
+    });
+
+}
+
+future<bytes> encryption::system_key::decrypt(const bytes& b) {
+    return get_key().then([b](shared_ptr<symmetric_key> k) {
+        auto i = k->iv_len();
+        bytes res(bytes::initialized_later(), b.size() - i);
+        auto n = k->decrypt(reinterpret_cast<const char*>(b.data()) + i,
+                        b.size() - i, reinterpret_cast<char*>(res.data()),
+                        res.size(), reinterpret_cast<const char*>(b.data()));
+        res.resize(n);
+        return make_ready_future<bytes>(std::move(res));
+    });
+}
+

ent/encryption/system_key.hh

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2015 ScyllaDB
+ *
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#pragma once
+
+#include "encryption.hh"
+#include "../../bytes.hh"
+
+namespace encryption {
+
+class symmetric_key;
+
+class system_key {
+public:
+    virtual ~system_key() {}
+    virtual future<shared_ptr<symmetric_key>> get_key() = 0;
+    virtual const sstring& name() const = 0;
+    virtual bool is_local() const = 0;
+    virtual future<> validate() const;
+
+    future<sstring> encrypt(const sstring&);
+    future<sstring> decrypt(const sstring&);
+    future<bytes> encrypt(const bytes&);
+    future<bytes> decrypt(const bytes&);
+};
+
+}
+

main.cc

@@ -119,6 +119,12 @@
 #include "message/dictionary_service.hh"
 #include "utils/disk_space_monitor.hh"
 
+
+#define P11_KIT_FUTURE_UNSTABLE_API
+extern "C" {
+#include <p11-kit/p11-kit.h>
+}
+
 seastar::metrics::metric_groups app_metrics;
 
 using namespace std::chrono_literals;
@@ -2415,6 +2421,7 @@ int main(int ac, char** av) {
         {"types", tools::scylla_types_main, "a command-line tool to examine values belonging to scylla types"},
         {"sstable", tools::scylla_sstable_main, "a multifunctional command-line tool to examine the content of sstables"},
         {"nodetool", tools::scylla_nodetool_main, "a command-line tool to administer local or remote ScyllaDB nodes"},
+        {"local-file-key-generator", tools::scylla_local_file_key_generator_main, "a command-line tool to generate encryption at rest keys"},
         {"perf-fast-forward", perf::scylla_fast_forward_main, "run performance tests by fast forwarding the reader on this server"},
         {"perf-row-cache-update", perf::scylla_row_cache_update_main, "run performance tests by updating row cache on this server"},
         {"perf-tablets", perf::scylla_tablets_main, "run performance tests of tablet metadata management"},
@@ -2472,5 +2479,20 @@ int main(int ac, char** av) {
         return 0;
     }
 
+    // We have to override p11-kit config path before p11-kit initialization.
+    // And the initialization will invoke on seastar initalization, so it has to
+    // be before app.run()
+    // #3583 - need to potentially ensure this for tools as well, since at least
+    // sstable* might need crypto libraries.
+    auto scylla_path = fs::read_symlink(fs::path("/proc/self/exe")); // could just be argv[0] I guess...
+    auto p11_modules = scylla_path.parent_path().parent_path().append("share/p11-kit/modules");
+    // Note: must be in scope for application lifetime. p11_kit_override_system_files does _not_
+    // copy input strings.
+    auto p11_modules_str = p11_modules.string<char>();
+    // #3392 only do this if we are actually packaged and the path exists.
+    if (fs::exists(p11_modules)) {
+        ::p11_kit_override_system_files(NULL, NULL, p11_modules_str.c_str(), NULL, NULL);
+    }
+
     return main_func(ac, av);
 }

schema/schema.hh

@@ -473,6 +473,9 @@ class partition_slice;
 class schema_extension {
 public:
     virtual ~schema_extension() {};
+    virtual future<> validate(const schema&) const {
+        return make_ready_future<>();
+    }
     virtual bytes serialize() const = 0;
     virtual bool is_placeholder() const {
         return false;

service/migration_manager.cc

@@ -626,6 +626,13 @@ std::vector<mutation> prepare_new_keyspace_announcement(replica::database& db, l
     return db::schema_tables::make_create_keyspace_mutations(db.features().cluster_schema_features(), ksm, timestamp);
 }
 
+static
+future<> validate(schema_ptr schema) {
+    return do_for_each(schema->extensions(), [schema](auto & p) {
+        return p.second->validate(*schema);
+    });
+}
+
 static future<std::vector<mutation>> include_keyspace(
         storage_proxy& sp, const keyspace_metadata& keyspace, std::vector<mutation> mutations) {
     // Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
@@ -656,6 +663,7 @@ static future<std::vector<mutation>> do_prepare_new_column_family_announcement(s
 }
 
 future<std::vector<mutation>> prepare_new_column_family_announcement(storage_proxy& sp, schema_ptr cfm, api::timestamp_type timestamp) {
+  return validate(cfm).then([&sp, cfm, timestamp] {
     try {
         auto& db = sp.get_db().local();
         auto ksm = db.find_keyspace(cfm->ks_name()).metadata();
@@ -663,6 +671,7 @@ future<std::vector<mutation>> prepare_new_column_family_announcement(storage_pro
     } catch (const replica::no_such_keyspace& e) {
         throw exceptions::configuration_exception(format("Cannot add table '{}' to non existing keyspace '{}'.", cfm->cf_name(), cfm->ks_name()));
     }
+  });
 }
 
 future<> prepare_new_column_family_announcement(std::vector<mutation>& mutations,
@@ -677,6 +686,7 @@ future<> prepare_new_column_family_announcement(std::vector<mutation>& mutations
 future<std::vector<mutation>> prepare_column_family_update_announcement(storage_proxy& sp,
         schema_ptr cfm, std::vector<view_ptr> view_updates, api::timestamp_type ts) {
     warn(unimplemented::cause::VALIDATION);
+    co_await validate(cfm);
     try {
         auto& db = sp.local_db();
         auto&& old_schema = db.find_column_family(cfm->ks_name(), cfm->cf_name()).schema(); // FIXME: Should we lookup by id?
@@ -826,6 +836,7 @@ future<std::vector<mutation>> prepare_type_drop_announcement(storage_proxy& sp,
 }
 
 future<std::vector<mutation>> prepare_new_view_announcement(storage_proxy& sp, view_ptr view, api::timestamp_type ts) {
+  return validate(view).then([&sp, view = std::move(view), ts] {
     auto& db = sp.local_db();
     try {
         auto keyspace = db.find_keyspace(view->ks_name()).metadata();
@@ -846,9 +857,11 @@ future<std::vector<mutation>> prepare_new_view_announcement(storage_proxy& sp, v
         return make_exception_future<std::vector<mutation>>(
             exceptions::configuration_exception(format("Cannot add view '{}' to non existing keyspace '{}'.", view->cf_name(), view->ks_name())));
     }
+  });
 }
 
 future<std::vector<mutation>> prepare_view_update_announcement(storage_proxy& sp, view_ptr view, api::timestamp_type ts) {
+    co_await validate(view);
     auto db = sp.data_dictionary();
     try {
         auto&& keyspace = db.find_keyspace(view->ks_name()).metadata();

sstables/sstables.hh

@@ -783,6 +783,10 @@ private:
 public:
     future<> read_toc() noexcept;
 
+    shareable_components& get_shared_components() const {
+        return *_components;
+    }
+
     schema_ptr get_schema() const {
         return _schema;
     }

test/boost/CMakeLists.txt

@@ -80,6 +80,15 @@ add_scylla_test(duration_test
 add_scylla_test(dynamic_bitset_test
   KIND BOOST
   LIBRARIES utils)
+add_scylla_test(encrypted_file_test
+  KIND SEASTAR
+  LIBRARIES
+    encryption)
+add_scylla_test(encryption_at_rest_test
+  KIND SEASTAR
+  LIBRARIES
+    Boost::filesystem
+    encryption)
 add_scylla_test(enum_option_test
   KIND BOOST)
 add_scylla_test(enum_set_test

test/boost/encrypted_file_test.cc

@@ -0,0 +1,265 @@
+/*
+ * Copyright (C) 2016 ScyllaDB
+ */
+
+
+
+#include <boost/range/irange.hpp>
+#include <boost/range/adaptors.hpp>
+#include <boost/range/algorithm.hpp>
+#include <boost/test/unit_test.hpp>
+#include <stdint.h>
+#include <random>
+
+#include <seastar/core/future-util.hh>
+#include <seastar/core/seastar.hh>
+#include <seastar/core/shared_ptr.hh>
+#include <seastar/core/thread.hh>
+
+#include <seastar/testing/test_case.hh>
+
+#include "ent/encryption/encryption.hh"
+#include "ent/encryption/symmetric_key.hh"
+#include "ent/encryption/encrypted_file_impl.hh"
+#include "test/lib/tmpdir.hh"
+#include "test/lib/random_utils.hh"
+#include "test/lib/exception_utils.hh"
+
+using namespace encryption;
+
+static tmpdir dir;
+
+static future<std::tuple<file, ::shared_ptr<symmetric_key>>> make_file(const sstring& name, open_flags mode, ::shared_ptr<symmetric_key> k = nullptr) {
+    file f = co_await open_file_dma(sstring(dir.path() / std::string(name)), mode);
+    if (k == nullptr) {
+        key_info info{"AES/CBC", 256};
+        k = ::make_shared<symmetric_key>(info);
+    }
+    co_return std::tuple(file(make_encrypted_file(f, k)), k);
+}
+
+static temporary_buffer<uint8_t> generate_random(size_t n, size_t align) {
+    auto tmp = temporary_buffer<uint8_t>::aligned(align, align_up(n, align));
+    auto data = tests::random::get_sstring(n);
+    std::copy(data.begin(), data.end(), tmp.get_write());
+    return tmp;
+}
+
+static future<> test_random_data_disk(size_t n) {
+    auto name = "test_rand_" + std::to_string(n);
+    auto t = co_await make_file(name, open_flags::rw|open_flags::create);
+    auto f = std::get<0>(t);
+    std::exception_ptr ex = nullptr;
+
+    try {
+        auto k = std::get<1>(t);
+        auto a = f.memory_dma_alignment();
+        auto buf = generate_random(n, a);
+        auto w = co_await f.dma_write(0, buf.get(), buf.size());
+
+        co_await f.flush();
+        if (n != buf.size()) {
+            co_await f.truncate(n);
+        }
+
+        BOOST_REQUIRE_EQUAL(w, buf.size());
+
+        auto k2 = ::make_shared<symmetric_key>(k->info(), k->key());
+        auto f2 = std::get<0>(co_await make_file(name, open_flags::ro, k2));
+
+        auto tmp = temporary_buffer<uint8_t>::aligned(a, buf.size());
+        auto n2 = co_await f2.dma_read(0, tmp.get_write(), tmp.size());
+
+        BOOST_REQUIRE_EQUAL(n2, n);
+        BOOST_REQUIRE_EQUAL_COLLECTIONS(tmp.get(), tmp.get() + n2, buf.get(), buf.get() + n2);
+    } catch (...) {
+        ex = std::current_exception();
+    }
+
+    co_await f.close();
+    if (ex) {
+        std::rethrow_exception(ex);
+    }
+}
+
+static void test_random_data(size_t n) {
+    auto buf = generate_random(n, 8);
+
+
+    // first, verify padded.
+    {
+        key_info info{"AES/CBC/PKCSPadding", 256};
+        auto k = ::make_shared<symmetric_key>(info);
+
+        bytes b(bytes::initialized_later(), k->iv_len());
+        k->generate_iv(b.data(), k->iv_len());
+
+        temporary_buffer<uint8_t> tmp(n + k->block_size());
+        k->encrypt(buf.get(), buf.size(), tmp.get_write(), tmp.size(), b.data());
+
+        auto bytes = k->key();
+        auto k2 = ::make_shared<symmetric_key>(info, bytes);
+
+        temporary_buffer<uint8_t> tmp2(n + k->block_size());
+        k2->decrypt(tmp.get(), tmp.size(), tmp2.get_write(), tmp2.size(), b.data());
+
+        BOOST_REQUIRE_EQUAL_COLLECTIONS(tmp2.get(), tmp2.get() + n, buf.get(), buf.get() + n);
+    }
+
+    // unpadded
+    {
+        key_info info{"AES/CBC", 256};
+        auto k = ::make_shared<symmetric_key>(info);
+
+        bytes b(bytes::initialized_later(), k->iv_len());
+        k->generate_iv(b.data(), k->iv_len());
+
+        temporary_buffer<uint8_t> tmp(n);
+        k->encrypt_unpadded(buf.get(), buf.size(), tmp.get_write(), b.data());
+
+        auto bytes = k->key();
+        auto k2 = ::make_shared<symmetric_key>(info, bytes);
+
+        temporary_buffer<uint8_t> tmp2(buf.size());
+        k2->decrypt_unpadded(tmp.get(), tmp.size(), tmp2.get_write(), b.data());
+
+        BOOST_REQUIRE_EQUAL_COLLECTIONS(tmp2.get(), tmp2.get() + n, buf.get(), buf.get() + n);
+    }
+}
+
+
+BOOST_AUTO_TEST_CASE(test_encrypting_data_128) {
+    test_random_data(128);
+}
+
+BOOST_AUTO_TEST_CASE(test_encrypting_data_4k) {
+    test_random_data(4*1024);
+}
+
+
+SEASTAR_TEST_CASE(test_encrypted_file_data_4k) {
+    return test_random_data_disk(4*1024);
+}
+
+SEASTAR_TEST_CASE(test_encrypted_file_data_16k) {
+    return test_random_data_disk(16*1024);
+}
+
+SEASTAR_TEST_CASE(test_encrypted_file_data_unaligned) {
+    return test_random_data_disk(16*1024 - 3);
+}
+
+SEASTAR_TEST_CASE(test_encrypted_file_data_unaligned2) {
+    return test_random_data_disk(16*1024 - 4092);
+}
+
+SEASTAR_TEST_CASE(test_short) {
+    auto name = "test_short";
+    file f = co_await open_file_dma(sstring(dir.path() / name), open_flags::rw|open_flags::create);
+    co_await f.truncate(1);
+    co_await f.close();
+
+    auto t = co_await make_file(name, open_flags::ro);
+    f = std::get<0>(t);
+    std::exception_ptr ex = nullptr;
+
+    try {
+        temporary_buffer<char> buf(f.memory_dma_alignment());
+
+        BOOST_REQUIRE_EXCEPTION(
+            co_await f.dma_read(0, buf.get_write(), buf.size()),
+            std::domain_error,
+            exception_predicate::message_contains("file size 1, expected 0 or at least 16")
+        );
+    } catch (...) {
+        ex = std::current_exception();
+    }
+
+    co_await f.close();
+    if (ex) {
+        std::rethrow_exception(ex);
+    }
+}
+
+SEASTAR_TEST_CASE(test_truncating_empty) {
+    auto name = "test_truncating_empty";
+    auto t = co_await make_file(name, open_flags::rw|open_flags::create);
+    auto f = std::get<0>(t);
+    auto k = std::get<1>(t);
+    auto s = 64 * f.memory_dma_alignment();
+
+    co_await f.truncate(s);
+
+    temporary_buffer<char> buf(s);
+    auto n = co_await f.dma_read(0, buf.get_write(), buf.size());
+
+    co_await f.close();
+
+    BOOST_REQUIRE_EQUAL(s, n);
+
+    for (auto c : buf) {
+        BOOST_REQUIRE_EQUAL(c, 0);
+    }
+}
+
+SEASTAR_TEST_CASE(test_truncating_extend) {
+    auto name = "test_truncating_extend";
+    auto t = co_await make_file(name, open_flags::rw|open_flags::create);
+    auto f = std::get<0>(t);
+    auto k = std::get<1>(t);
+    auto a = f.memory_dma_alignment();
+    auto s = 32 * a;
+    auto buf = generate_random(s, a);
+    auto w = co_await f.dma_write(0, buf.get(), buf.size());
+
+    co_await f.flush();
+    BOOST_REQUIRE_EQUAL(s, w);
+
+    for (size_t i = 1; i < 64; ++i) {
+        // truncate smaller, unaligned
+        auto l = w - i;
+        auto r = w + 8 * a;
+        co_await f.truncate(l);
+        BOOST_REQUIRE_EQUAL(l, (co_await f.stat()).st_size);
+
+        {
+            auto tmp = temporary_buffer<uint8_t>::aligned(a, align_up(l, a));
+            auto n = co_await f.dma_read(0, tmp.get_write(), tmp.size());
+
+            BOOST_REQUIRE_EQUAL(l, n);
+            BOOST_REQUIRE_EQUAL_COLLECTIONS(tmp.get(), tmp.get() + l, buf.get(), buf.get() + l);
+
+            auto k = align_down(l, a);
+
+            while (k > 0) {
+                n = co_await f.dma_read(0, tmp.get_write(), k);
+
+                BOOST_REQUIRE_EQUAL(k, n);
+                BOOST_REQUIRE_EQUAL_COLLECTIONS(tmp.get(), tmp.get() + k, buf.get(), buf.get() + k);
+
+                n = co_await f.dma_read(k, tmp.get_write(), tmp.size());
+                BOOST_REQUIRE_EQUAL(l - k, n);
+                BOOST_REQUIRE_EQUAL_COLLECTIONS(tmp.get(), tmp.get() + n, buf.get() + k, buf.get() + k + n);
+
+                k -= a;
+            }
+        }
+
+        co_await f.truncate(r);
+        BOOST_REQUIRE_EQUAL(r, (co_await f.stat()).st_size);
+
+        auto tmp = temporary_buffer<uint8_t>::aligned(a, align_up(r, a));
+        auto n = co_await f.dma_read(0, tmp.get_write(), tmp.size());
+
+        BOOST_REQUIRE_EQUAL(r, n);
+        BOOST_REQUIRE_EQUAL_COLLECTIONS(tmp.get(), tmp.get() + l, buf.get(), buf.get() + l);
+
+        while (l < r) {
+            BOOST_REQUIRE_EQUAL(tmp[l], 0);
+            ++l;
+        }
+    }
+
+    co_await f.close();
+}
+

test/boost/kmip_wrapper.py

@@ -0,0 +1,95 @@
+import ssl
+import sys
+
+from kmip.services import auth
+from kmip.services.server.server import build_argument_parser
+from kmip.services.server.server import KmipServer
+
+# Helper wrapper for running pykmip in scylla testing. Needed because TLS options
+# (hardcoded) in pykmip are obsolete and will not work with connecting using gnutls
+# of any modern variety.
+
+class TLS13AuthenticationSuite(auth.TLS12AuthenticationSuite):
+    """
+    An authentication suite used to establish secure network connections.
+
+    Supports TLS 1.3. More importantly, works with gnutls-<recent>
+    """
+    def __init__(self, cipher_suites=None):
+        """
+        Create a TLS12AuthenticationSuite object.
+
+        Args:
+            cipher_suites (list): A list of strings representing the names of
+                cipher suites to use. Overrides the default set of cipher
+                suites. Optional, defaults to None.
+        """
+        super().__init__(cipher_suites)
+        self._protocol = ssl.PROTOCOL_TLS_SERVER
+
+def main():
+    # Build argument parser and parser command-line arguments.
+    parser = build_argument_parser()
+    opts, args = parser.parse_args(sys.argv[1:])
+
+    kwargs = {}
+    if opts.hostname:
+        kwargs['hostname'] = opts.hostname
+    if opts.port:
+        kwargs['port'] = opts.port
+    if opts.certificate_path:
+        kwargs['certificate_path'] = opts.certificate_path
+    if opts.key_path:
+        kwargs['key_path'] = opts.key_path
+    if opts.ca_path:
+        kwargs['ca_path'] = opts.ca_path
+    if opts.auth_suite:
+        kwargs['auth_suite'] = opts.auth_suite
+    if opts.config_path:
+        kwargs['config_path'] = opts.config_path
+    if opts.log_path:
+        kwargs['log_path'] = opts.log_path
+    if opts.policy_path:
+        kwargs['policy_path'] = opts.policy_path
+    if opts.ignore_tls_client_auth:
+        kwargs['enable_tls_client_auth'] = False
+    if opts.logging_level:
+        kwargs['logging_level'] = opts.logging_level
+    if opts.database_path:
+        kwargs['database_path'] = opts.database_path
+
+    kwargs['live_policies'] = True
+
+    # Create and start the server.
+    s = KmipServer(**kwargs)
+    # Fix TLS. Try to get this into mainline project, but that will take time...
+    s.auth_suite = TLS13AuthenticationSuite(s.auth_suite.ciphers)
+    # force port to zero -> select dynamically
+    s.config.settings['port'] = 0
+
+    def fake_wrap_ssl(sock, keyfile=None, certfile=None,
+                server_side=False, cert_reqs=ssl.CERT_NONE,
+                ssl_version=ssl.PROTOCOL_TLS, ca_certs=None,
+                do_handshake_on_connect=True,
+                suppress_ragged_eofs=True,
+                ciphers=None):
+        ctxt = ssl.SSLContext(protocol = ssl_version)
+        ctxt.load_cert_chain(certfile=certfile, keyfile=keyfile)
+        ctxt.verify_mode = cert_reqs
+        ctxt.load_verify_locations(cafile=ca_certs)
+        ctxt.set_ciphers(ciphers)
+        return ctxt.wrap_socket(sock, server_side=server_side
+                              , do_handshake_on_connect=do_handshake_on_connect
+                              , suppress_ragged_eofs=suppress_ragged_eofs)
+
+    ssl.wrap_socket = fake_wrap_ssl
+
+    print("Starting...")
+
+    with s:
+        print("Listening on {}".format(s._socket.getsockname()[1]))
+        sys.stdout.flush()
+        s.serve()
+
+if __name__ == '__main__':
+    main()

test/boost/symmetric_key_test.cc

@@ -0,0 +1,221 @@
+/*
+ * Copyright (C) 2016 ScyllaDB
+ */
+
+
+
+#include <boost/range/irange.hpp>
+#include <boost/range/adaptors.hpp>
+#include <boost/range/algorithm.hpp>
+#include <boost/test/unit_test.hpp>
+#include <boost/lexical_cast.hpp>
+#include <stdint.h>
+#include <random>
+
+#include <seastar/core/future-util.hh>
+#include <seastar/core/seastar.hh>
+#include <seastar/core/shared_ptr.hh>
+#include <seastar/core/thread.hh>
+#include <seastar/core/align.hh>
+
+#include <seastar/testing/test_case.hh>
+
+#include "ent/encryption/encryption.hh"
+#include "ent/encryption/symmetric_key.hh"
+
+using namespace encryption;
+
+static temporary_buffer<uint8_t> generate_random(size_t n, size_t align) {
+    std::random_device r;
+    std::default_random_engine e1(r());
+    std::uniform_int_distribution<uint8_t> dist('0', 'z');
+
+    auto tmp = temporary_buffer<uint8_t>::aligned(align, align_up(n, align));
+    std::generate(tmp.get_write(), tmp.get_write() + tmp.size(), std::bind(dist, std::ref(e1)));
+    return tmp;
+}
+
+static void test_random_data(const sstring& desc, unsigned int bits) {
+    auto buf = generate_random(128, 8);
+    auto n = buf.size();
+
+    // first, verify padded.
+    {
+        key_info info{desc, bits};
+        auto k = ::make_shared<symmetric_key>(info);
+
+        bytes b(bytes::initialized_later(), k->iv_len());
+        k->generate_iv(b.data(), k->iv_len());
+
+        temporary_buffer<uint8_t> tmp(n + k->block_size());
+        k->encrypt(buf.get(), buf.size(), tmp.get_write(), tmp.size(), b.data());
+
+        auto bytes = k->key();
+        auto k2 = ::make_shared<symmetric_key>(info, bytes);
+
+        temporary_buffer<uint8_t> tmp2(n + k->block_size());
+        k2->decrypt(tmp.get(), tmp.size(), tmp2.get_write(), tmp2.size(), b.data());
+
+        BOOST_REQUIRE_EQUAL_COLLECTIONS(tmp2.get(), tmp2.get() + n, buf.get(), buf.get() + n);
+    }
+
+    // unpadded
+    {
+        auto desc2 = desc;
+        desc2.resize(desc.find_last_of('/'));
+        key_info info{desc2, bits};
+        auto k = ::make_shared<symmetric_key>(info);
+
+        bytes b(bytes::initialized_later(), k->iv_len());
+        k->generate_iv(b.data(), k->iv_len());
+
+        temporary_buffer<uint8_t> tmp(n);
+        k->encrypt_unpadded(buf.get(), buf.size(), tmp.get_write(), b.data());
+
+        auto bytes = k->key();
+        auto k2 = ::make_shared<symmetric_key>(info, bytes);
+
+        temporary_buffer<uint8_t> tmp2(buf.size());
+        k2->decrypt_unpadded(tmp.get(), tmp.size(), tmp2.get_write(), b.data());
+
+        BOOST_REQUIRE_EQUAL_COLLECTIONS(tmp2.get(), tmp2.get() + n, buf.get(), buf.get() + n);
+    }
+}
+
+
+SEASTAR_TEST_CASE(test_cipher_types) {
+    static const std::unordered_map<sstring, std::vector<unsigned int>> ciphers = {
+            { "AES/CBC/PKCS5Padding", { 128, 192, 256 } },
+            { "AES/ECB/PKCS5Padding", { 128, 192, 256 } },
+            { "DES/CBC/PKCS5Padding", { 56 } },
+            { "DESede/CBC/PKCS5Padding", { 112, 168 } },
+            { "Blowfish/CBC/PKCS5Padding", { 32, 64, 448 } },
+            { "RC2/CBC/PKCS5Padding", { 40, 41, 64, 67, 120, 128 } },
+    };
+
+    for (auto & p : ciphers) {
+        for (auto s : p.second) {
+            test_random_data(p.first, s);
+        }
+    }
+    return make_ready_future<>();
+}
+
+// OpenSSL only supports one form of padding. We used to just allow
+// non-empty string -> pkcs5/pcks7. We now instead verify this to be 
+// within the "sane" limits, i.e. pkcs, pkcs5 or pkcs7. 
+// Check an non-exhaustive set of invalid padding options and verify 
+// we get an exception as expected.
+// See below for test for valid strings.
+SEASTAR_TEST_CASE(test_invalid_padding_options) {
+    static const std::unordered_map<sstring, unsigned int> ciphers = {
+            { "AES/CBC/PKCSU", 128 },
+            { "AES/ECB/Gris", 128 },
+            { "DES/CBC/PKCS12Padding", 56 },
+            { "DES/CBC/KorvPadding", 56 },
+            { "DES/CBC/MUPadding", 56 },
+    };
+    for (auto& p : ciphers) {
+        try {
+            key_info info{p.first, p.second};
+            symmetric_key k(info);
+            BOOST_ERROR("should not reach");
+        } catch (...) {
+            // ok.
+        }
+    }
+    return make_ready_future<>();
+}
+
+SEASTAR_TEST_CASE(test_valid_padding_options) {
+    static const std::unordered_map<sstring, unsigned int> ciphers = {
+            { "AES/CBC/PKCS", 128 },
+            { "AES/CBC/PKCSPadding", 128 },
+            { "AES/ECB/PKCS7Padding", 128 },
+            { "AES/ECB/PKCS7", 128 },
+            { "DES/CBC/PKCS5Padding", 56 },
+            { "DES/CBC/PKCS5", 56 },
+            { "AES/CBC/NoPadding", 128 },
+            { "AES/ECB/NoPadding", 128 },
+            { "DES/CBC/NoPadding", 56 },
+            { "AES/CBC/No", 128 },
+            { "AES/ECB/No", 128 },
+            { "DES/CBC/No", 56 },
+    };
+    for (auto& p : ciphers) {
+        key_info info{p.first, p.second};
+        symmetric_key k(info);
+
+        auto errors = k.validate_exact_info_result();
+        BOOST_REQUIRE_EQUAL(errors, std::string{});
+    }
+    return make_ready_future<>();
+}
+
+SEASTAR_TEST_CASE(test_warn_adjusted_options) {
+    static const std::unordered_map<sstring, std::vector<unsigned int>> ciphers = {
+            // blowfish only supports CBC and will become CBC whatever you say
+            { "Blowfish/CFB/PKCS5Padding", { 32, 64, 448 } },
+            { "Blowfish/XTS/PKCS5Padding", { 32, 64, 448 } },
+    };
+    for (auto& p : ciphers) {
+        for (auto s : p.second) {
+            auto alg = p.first;
+            key_info info{alg, s};
+            symmetric_key k(info);
+
+            auto errors = k.validate_exact_info_result();
+            BOOST_REQUIRE_NE(errors, std::string{});
+        }
+    }
+    return make_ready_future<>();
+}
+
+/**
+ * Verifies that when using defaults in a key, the key info returned is still
+ * equal to the input one (by bit and textually)
+ */
+SEASTAR_TEST_CASE(test_cipher_defaults) {
+    static const std::unordered_map<sstring, std::vector<unsigned int>> ciphers = {
+            { "AES/CBC/PKCS5Padding", { 128, 192, 256 } },
+            { "AES/ECB/PKCS5Padding", { 128, 192, 256 } },
+            { "DES/CBC/PKCS5Padding", { 56 } },
+            { "DESede/CBC/PKCS5Padding", { 112, 168 } },
+            { "Blowfish/CBC/PKCS5Padding", { 32, 64, 448 } },
+            { "RC2/CBC/PKCS5Padding", { 40, 41, 64, 67, 120, 128 } },
+    };
+
+    for (auto& p : ciphers) {
+        for (auto s : p.second) {
+            auto alg = p.first;
+            for (;;) {
+                key_info info{alg, s};
+                symmetric_key k(info);
+
+                BOOST_REQUIRE_EQUAL(info, k.info());
+                BOOST_REQUIRE_EQUAL(boost::lexical_cast<std::string>(info), boost::lexical_cast<std::string>(k.info()));
+
+                auto i = alg.find_last_of('/');
+                if (i != sstring::npos) {
+                    alg.resize(i);
+                    continue;
+                }
+                // also verify that whatever we say (or don't say), we get a blockmode
+                // -> iv len > 0
+                BOOST_CHECK_GT(k.iv_len(), 0);
+
+                auto errors = k.validate_exact_info_result();
+                if (i != sstring::npos) {
+                    BOOST_REQUIRE_EQUAL(errors, std::string{});
+                } else {
+                    // Again, if we cut out block mode (i.e. only cipher name left)
+                    // we will still force a block mode. Thus this should warn.
+                    BOOST_REQUIRE_NE(errors, std::string{});
+                }
+
+                break;
+            }
+        }
+    }
+    return make_ready_future<>();
+}

test/lib/cql_test_env.cc

@@ -52,6 +52,7 @@
 #include "message/messaging_service.hh"
 #include "gms/gossiper.hh"
 #include "gms/feature_service.hh"
+#include "service/qos/service_level_controller.hh"
 #include "db/system_keyspace.hh"
 #include "db/system_distributed_keyspace.hh"
 #include "db/sstables-format-selector.hh"
@@ -174,8 +175,8 @@ private:
     struct core_local_state {
         service::client_state client_state;
 
-        core_local_state(auth::service& auth_service, qos::service_level_controller& sl_controller)
-            : client_state(service::client_state::external_tag{}, auth_service, &sl_controller, infinite_timeout_config)
+        core_local_state(auth::service& auth_service, qos::service_level_controller& sl_controller, timeout_config timeout)
+            : client_state(service::client_state::external_tag{}, auth_service, &sl_controller, timeout)
         {
             client_state.set_login(auth::authenticated_user(testing_superuser));
         }
@@ -1082,7 +1083,7 @@ private:
 
             _group0_client = &group0_client;
 
-            _core_local.start(std::ref(_auth_service), std::ref(_sl_controller)).get();
+            _core_local.start(std::ref(_auth_service), std::ref(_sl_controller), cfg_in.query_timeout.value_or(infinite_timeout_config)).get();
             auto stop_core_local = defer([this] { _core_local.stop().get(); });
 
             if (!local_db().has_keyspace(ks_name)) {

test/lib/cql_test_env.hh

@@ -98,6 +98,8 @@ public:
     bool ms_listen = false;
     bool run_with_raft_recovery = false;
 
+    std::optional<timeout_config> query_timeout;
+
     cql_test_config();
     cql_test_config(const cql_test_config&);
     cql_test_config(shared_ptr<db::config>);

test/lib/tmpdir.cc

@@ -31,9 +31,27 @@ tmpdir::sweeper::~sweeper() {
     }
 }
 
-tmpdir::tmpdir()
-    : _path(fs::temp_directory_path() / fs::path(fmt::format(FMT_STRING("scylla-{}"), utils::make_random_uuid()))) {
-    fs::create_directories(_path);
+tmpdir::tmpdir() {
+    auto tmp = fs::temp_directory_path();
+    for (;;) {
+        // Reduce the path length of the created tmp dir. This might seem
+        // silly when running with base TMPDIR=/tmp or similar, but
+        // in a lot of CI testing, TMPDIR will be a loooooong path into
+        // jenkins workdirs or similar -> this path will be 100+ chars long.
+        // Again, this should most often not be a problem, _but_ if we
+        // for example run something like a sub process of a python server,
+        // which will try to create various unix sockets et al for its
+        // operations, the TMPDIR base for this must not exceed 107 chars.
+        // Note: converting UUID to string first, because for some reason
+        // our UUID formatter does not respect width/precision. Feel free to
+        // change once it does.
+        _path = tmp / fmt::format("scylla-{:.8}", fmt::to_string(utils::make_random_uuid()));
+        // Note: this is a slight improvement also, in that we ensure the dir 
+        // we use is actually created by us.
+        if (fs::create_directories(_path)) {
+            break;
+        }
+    }
 }
 
 tmpdir::tmpdir(tmpdir&& other) noexcept : _path(std::exchange(other._path, {})) {}

test/resource/certs/cacert.pem

@@ -0,0 +1,82 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 1572441855 (0x5db98eff)
+    Signature Algorithm: sha256WithRSAEncryption
+        Issuer: C=US, O=HyTrust Inc., CN=HyTrust KeyControl Certificate Authority
+        Validity
+            Not Before: Jun  1 00:00:00 2011 GMT
+            Not After : Dec 31 23:59:59 2049 GMT
+        Subject: C=US, O=HyTrust Inc., CN=HyTrust KeyControl Certificate Authority
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                Public-Key: (2048 bit)
+                Modulus:
+                    00:ce:97:5b:a1:30:b1:26:00:31:8a:aa:bd:a2:0c:
+                    19:1b:24:83:05:20:7a:63:ac:6c:ab:0f:80:24:47:
+                    2b:03:94:86:25:a4:10:51:fb:b6:e8:5e:33:db:e2:
+                    91:98:f3:2a:b1:78:ef:83:a5:f2:e1:79:36:44:06:
+                    3b:01:cd:1a:47:c1:74:89:2b:d3:3c:8d:f1:fb:c2:
+                    90:88:fe:18:d9:81:7b:2a:e4:67:61:87:17:23:38:
+                    29:5a:66:eb:d0:01:a6:da:c3:a5:7f:f5:d1:9e:f5:
+                    04:dc:1c:4a:62:2e:e2:5b:5f:22:56:61:fe:ba:66:
+                    c2:ad:a9:51:43:9c:28:e4:8f:fa:05:12:fa:0d:a5:
+                    35:e3:2f:99:e8:a4:98:09:f9:e7:c8:e0:6c:a9:bd:
+                    e9:59:b0:83:07:09:10:10:5b:aa:b5:72:3b:40:e6:
+                    38:f4:e3:f8:9a:55:8e:5e:ae:5c:3e:c3:08:34:13:
+                    9c:19:fc:65:07:ac:3f:98:ae:a0:d2:f8:1d:4c:bf:
+                    cb:93:a7:e4:d6:37:84:0a:0c:3a:1f:86:f2:35:0c:
+                    2e:66:b0:9b:43:8e:bc:e4:b9:b0:bf:33:67:c2:97:
+                    df:47:6c:65:cc:55:38:70:a9:39:27:60:e1:74:14:
+                    34:e9:a6:a0:b6:11:de:61:94:9a:6b:83:f1:84:8d:
+                    27:9f
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Subject Key Identifier:
+                66:AD:DE:45:50:5D:54:68:1F:B0:56:00:65:FB:D1:F2:97:57:EF:6E
+            X509v3 Authority Key Identifier:
+                keyid:66:AD:DE:45:50:5D:54:68:1F:B0:56:00:65:FB:D1:F2:97:57:EF:6E
+                DirName:/C=US/O=HyTrust Inc./CN=HyTrust KeyControl Certificate Authority
+                serial:5D:B9:8E:FF
+
+            X509v3 Basic Constraints:
+                CA:TRUE
+    Signature Algorithm: sha256WithRSAEncryption
+         ab:a9:19:07:07:f0:b9:74:e1:a8:49:db:bd:c1:21:fc:38:38:
+         79:dd:2f:3e:59:be:96:79:1d:18:d4:5e:1f:31:47:fb:bd:d4:
+         96:d7:be:87:7e:0d:e4:9e:7c:7a:36:c9:9a:5f:e5:63:38:33:
+         68:cf:b1:92:d0:b8:81:1a:6f:23:27:d8:71:50:41:63:ce:5f:
+         20:69:72:4c:cd:07:ab:35:58:fe:da:d5:26:1e:44:f4:97:e3:
+         ff:6c:80:db:31:17:13:52:6c:fb:68:34:71:11:af:b6:84:3b:
+         b1:5c:d3:67:25:e1:5a:31:a6:68:83:ec:c4:3e:e8:f6:08:60:
+         d0:2a:26:9e:fe:07:08:57:6e:9a:dd:6e:ba:a2:10:ab:2e:fd:
+         cd:52:a3:2f:e0:59:6d:33:39:05:ed:fd:ed:ac:b0:e7:98:5e:
+         f2:51:00:12:df:4c:8a:0c:e2:11:df:43:65:d0:f3:a1:85:59:
+         6d:d4:bb:a0:97:f7:c7:40:63:b3:24:cf:ec:5e:9e:42:1b:cc:
+         e2:36:43:d1:83:79:11:48:3b:3d:db:c3:2a:03:4f:cd:53:2d:
+         07:8d:0e:28:4a:a9:58:e0:27:c3:47:f6:ab:00:cd:fc:31:ed:
+         99:b9:57:2e:2d:5a:79:5f:48:14:39:8b:0e:da:1c:a0:d6:4e:
+         d4:81:83:49
+-----BEGIN CERTIFICATE-----
+MIID4jCCAsqgAwIBAgIEXbmO/zANBgkqhkiG9w0BAQsFADBXMQswCQYDVQQGEwJV
+UzEVMBMGA1UEChMMSHlUcnVzdCBJbmMuMTEwLwYDVQQDEyhIeVRydXN0IEtleUNv
+bnRyb2wgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MB4XDTExMDYwMTAwMDAwMFoXDTQ5
+MTIzMTIzNTk1OVowVzELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEh5VHJ1c3QgSW5j
+LjExMC8GA1UEAxMoSHlUcnVzdCBLZXlDb250cm9sIENlcnRpZmljYXRlIEF1dGhv
+cml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM6XW6EwsSYAMYqq
+vaIMGRskgwUgemOsbKsPgCRHKwOUhiWkEFH7tuheM9vikZjzKrF474Ol8uF5NkQG
+OwHNGkfBdIkr0zyN8fvCkIj+GNmBeyrkZ2GHFyM4KVpm69ABptrDpX/10Z71BNwc
+SmIu4ltfIlZh/rpmwq2pUUOcKOSP+gUS+g2lNeMvmeikmAn558jgbKm96VmwgwcJ
+EBBbqrVyO0DmOPTj+JpVjl6uXD7DCDQTnBn8ZQesP5iuoNL4HUy/y5On5NY3hAoM
+Oh+G8jUMLmawm0OOvOS5sL8zZ8KX30dsZcxVOHCpOSdg4XQUNOmmoLYR3mGUmmuD
+8YSNJ58CAwEAAaOBtTCBsjAdBgNVHQ4EFgQUZq3eRVBdVGgfsFYAZfvR8pdX724w
+gYIGA1UdIwR7MHmAFGat3kVQXVRoH7BWAGX70fKXV+9uoVukWTBXMQswCQYDVQQG
+EwJVUzEVMBMGA1UEChMMSHlUcnVzdCBJbmMuMTEwLwYDVQQDEyhIeVRydXN0IEtl
+eUNvbnRyb2wgQ2VydGlmaWNhdGUgQXV0aG9yaXR5ggRduY7/MAwGA1UdEwQFMAMB
+Af8wDQYJKoZIhvcNAQELBQADggEBAKupGQcH8Ll04ahJ273BIfw4OHndLz5ZvpZ5
+HRjUXh8xR/u91JbXvod+DeSefHo2yZpf5WM4M2jPsZLQuIEabyMn2HFQQWPOXyBp
+ckzNB6s1WP7a1SYeRPSX4/9sgNsxFxNSbPtoNHERr7aEO7Fc02cl4VoxpmiD7MQ+
+6PYIYNAqJp7+BwhXbprdbrqiEKsu/c1Soy/gWW0zOQXt/e2ssOeYXvJRABLfTIoM
+4hHfQ2XQ86GFWW3Uu6CX98dAY7Mkz+xenkIbzOI2Q9GDeRFIOz3bwyoDT81TLQeN
+DihKqVjgJ8NH9qsAzfwx7Zm5Vy4tWnlfSBQ5iw7aHKDWTtSBg0k=
+-----END CERTIFICATE-----

test/resource/certs/scylla.pem

@@ -0,0 +1,57 @@
+Bag Attributes
+    localKeyID: 12 1A 34 A0 C8 58 91 A4 E3 B6 7F 16 F0 31 05 81 AA 27 82 05
+subject=/C=US/O=HyTrust Inc./CN=scylla
+issuer=/C=US/O=HyTrust Inc./CN=HyTrust KeyControl Certificate Authority
+-----BEGIN CERTIFICATE-----
+MIIDlTCCAn2gAwIBAgIFAPm5jwcwDQYJKoZIhvcNAQELBQAwVzELMAkGA1UEBhMC
+VVMxFTATBgNVBAoTDEh5VHJ1c3QgSW5jLjExMC8GA1UEAxMoSHlUcnVzdCBLZXlD
+b250cm9sIENlcnRpZmljYXRlIEF1dGhvcml0eTAgFw0yMTAxMDQxMjMwMTlaGA8y
+MDk5MDEwMTEyMzAxOVowNTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEh5VHJ1c3Qg
+SW5jLjEPMA0GA1UEAxMGc2N5bGxhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAosGginu/B4eQKyB26ar6rqg8FTMoCTf56iBwQnrOt17YN++IitlEhFyB
+X0QdA0z97jLIBckTCbieTg+CdtWDeCnM8IAjoN55C0Z2zBKH6cuPTTnDu0WaZY/8
+IafGwxcWllYpgQ3AiJFNK66QRIiiX+ejrS3+Co0PYWPzmSczWoxBgFhnXnTPE4ki
+MLvZ+zY1iXt83NbQIw8yMUcL+RYK4RlACf3bPztOss98LmyntIkNkZL8GblLoZbc
+AZc6udnDe1GuP+NlMO+1jPmyND/xz/kK2hkU4+yotBWVxM1lwpANnElsAHaRvthP
+kGsjoVvZgEgg6MQX+iaDngjmiLtUlwIDAQABo4GHMIGEMAkGA1UdEwQCMAAwLAYJ
+YIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1Ud
+DgQWBBTAgszBvUAggtuLWf2XucTzjLQeqDAfBgNVHSMEGDAWgBRmrd5FUF1UaB+w
+VgBl+9Hyl1fvbjAJBgNVHREEAjAAMA0GCSqGSIb3DQEBCwUAA4IBAQBJA9q7q5XL
+H7O/EiJYagWaqrO+AGVrg3DtkP0NTaumM/zYoOq9klZNwvsGZ88+L0hBQ8fj8O/l
+rTobdM7eT5p4JiwfN+MB8zXuZ+XjL+kIpqFqPJdVDtBLPGINHP7itrUo9Uk/9XcW
+JqPAwufEMN5X7iwN80aUVj6/iUiQ1yqXKQmERdoIPqiyHOEBwobwIAQ1bQKtEiBT
+ZE0hdDdI+ZCrtJYOES4kpR4WI3997doVhEusNGBETCWMm2HoJ8xsk7fgVgnpYlat
+95tGZ/ZR1Zaa+fXSm9adxJDCviG77pTZBa7nbzPoyG3wm76cTGSeHso8rvGVHsTh
+AraNJqMQkZT5
+-----END CERTIFICATE-----
+Bag Attributes
+    localKeyID: 12 1A 34 A0 C8 58 91 A4 E3 B6 7F 16 F0 31 05 81 AA 27 82 05
+Key Attributes: <No Attributes>
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCiwaCKe78Hh5Ar
+IHbpqvquqDwVMygJN/nqIHBCes63Xtg374iK2USEXIFfRB0DTP3uMsgFyRMJuJ5O
+D4J21YN4KczwgCOg3nkLRnbMEofpy49NOcO7RZplj/whp8bDFxaWVimBDcCIkU0r
+rpBEiKJf56OtLf4KjQ9hY/OZJzNajEGAWGdedM8TiSIwu9n7NjWJe3zc1tAjDzIx
+Rwv5FgrhGUAJ/ds/O06yz3wubKe0iQ2RkvwZuUuhltwBlzq52cN7Ua4/42Uw77WM
++bI0P/HP+QraGRTj7Ki0FZXEzWXCkA2cSWwAdpG+2E+QayOhW9mASCDoxBf6JoOe
+COaIu1SXAgMBAAECggEAefvDdmd+6obJH/mqBkIWtpbyyTTZOeeRUM/VM45VpovY
+ZDwMS3zB5K5sbFlhoVqwKzo2PlfRBAUx6PGo56XpbkNuDYcMrQJxGGlMmnD9GtZt
+ZgT4VUC6kopS/2p/BzMjw7N6UfZbqj+05htkl2kMwfKb8y04bPICaA0Aw3XlAGRS
+WIrfsXW32/hStXJvt8YSveIUP/WbdyEefwcKsu44LFvaq+j85KH75KLyZQre6n/A
+7A67WIs1AcjiBHq/nupNF9rZUFm5YzL/kOC+g9Klq6bIIlSu+2ULIjL0WLy5be8R
+OdIipE+ENa5HpC5J8z9WfGCf8yTB8/V+pt2dxZ9T6QKBgQDV2R1/9IgphiNCjPSE
+M7JbY+K0XJ5Mfq64h1apLjycsq93dUX+1OI/e/qlxKS6SpqCgEajKch/x16ylZeQ
+BNuG3zBuFPJH/17MfjdWFAW5ZJmbsnM05u+IU4aN0+r8laddvyNcdgLS4O6WvGrL
+z5Nxio0YLkZnCFBsjdUb5Fs00wKBgQDC1mViZp4yT1b21r070sYQ1w4wgLYfVNIF
+rc8AFevGh35kFRe2fFWl7T28Zqhetj7ES2LK7H93o8GdT28RuHrwst/nYL+lClmn
+37iHPuHwlqDussCxg5A49HWq2qEoZ5DF379b8FJgcBYxYBsu7LSmQNEgFtCdvD7y
+8C3uHieWrQKBgQCoOK0OFOxvzcc6+Or4fDpXzhFuVFVqU1Vab4xqdabUlXOWgzhW
+qFx6GCsp77CtozY8ZnAqthm0+r6xuR+K+Wc/h57vWabloCuQrdEV85Y1Kr/zTMN3
+4BqZoSr9srDtlUQdaNiGSYcbIDpPdVSFk3qnHJi1ZuGW92Fco4367P4aZQKBgFyZ
+4V2/P/jRVJfEv/Oq3ZArZgcPZX/GpHsmfHeh84lL5HpUvAxzU5GlC+57LBK3s2VA
+HxgrBvopzl+h3Twi3euAWIJzrSIXpTzwS5eb/26FaL+KHaNA0E8BgNtPRcEaV+hz
+y1M7CSvkmeelscW/oqxRqhMCROxzB8gW9v1xP4eRAoGADhZOjOmsfbBD58ijzaOJ
+i2bOjCPzkRsS0cxnVlILA7kWU9bPxOkHKmkZnaYIBJhAJsW9l1XA3mlS5gqf93EW
+VivIJZb10wI/MeRuITxanWiHPDcbbyOjBgaj6gNBbyrS9rDordYdLRVOsmlwvvJT
+a60BCRcu0IgS6uM0sKaojoc=
+-----END PRIVATE KEY-----

tools/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_library(tools STATIC)
 target_sources(tools
   PRIVATE
+    scylla-local-file-key-generator.cc
     load_system_tablets.cc
     read_mutation.cc
     scylla-types.cc

tools/entry_point.hh

@@ -10,6 +10,7 @@
 
 namespace tools {
 
+int scylla_local_file_key_generator_main(int argc, char** argv);
 int scylla_types_main(int argc, char** argv);
 int scylla_sstable_main(int argc, char** argv);
 int scylla_nodetool_main(int argc, char** argv);

tools/scylla-local-file-key-generator.cc

@@ -0,0 +1,166 @@
+/*
+ * Copyright (C) 2020-present ScyllaDB
+ */
+
+/*
+ * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+ */
+
+#include <seastar/core/coroutine.hh>
+#include <filesystem>
+#include <iostream>
+#include <ranges>
+
+#include <fmt/ranges.h>
+#include "compound.hh"
+#include "db/marshal/type_parser.hh"
+#include "schema/schema_builder.hh"
+#include "tools/utils.hh"
+#include "dht/i_partitioner.hh"
+#include "utils/managed_bytes.hh"
+#include "ent/encryption/symmetric_key.hh"
+#include "ent/encryption/local_file_provider.hh"
+
+using namespace seastar;
+using namespace tools::utils;
+
+namespace bpo = boost::program_options;
+
+namespace std {
+// required by boost::lexical_cast<std::string>(vector<string>), which is in turn used
+// by boost::program_option for printing out the default value of an option
+static std::ostream& operator<<(std::ostream& os, const std::vector<std::string>& v) {
+    return os << fmt::format("{}", v);
+}
+}
+
+namespace {
+
+const auto app_name = "local-file-key-generator";
+
+const std::vector<operation_option> global_options{
+    typed_option<std::string>("alg,a", "AES", "Key algorithm (i.e. AES, 3DES)"),
+    typed_option<std::string>("block-mode,b", "CBC", "Algorithm block mode (i.e. CBC, EBC)"),
+    typed_option<std::string>("padding,p", "PKCS5", "Algorithm padding method (i.e. PKCS5)"),
+    typed_option<unsigned>("length,l", 128, "Key length in bits (i.e. 128, 256)"),
+};
+
+const std::vector<operation_option> global_positional_options{
+    typed_option<std::vector<std::string>>("files", "key path|key name", -1),
+};
+
+const std::vector<operation> operations = {
+    {"generate", "creates a new key and stores to a new file",
+R"(
+Generate a key suitable for a given algorithm and key length 
+and store to a file readable by scylla encryption at rest 
+local file key provider.
+)"},
+    {"append", "same as generate, but appends key to existing file",
+R"(
+Generate a key suitable for a given algorithm and key length 
+and append to an existing file readable by scylla encryption at rest 
+local file key provider.
+)"},
+};
+
+}
+
+namespace tools {
+
+using namespace encryption;
+using namespace std::string_literals;
+namespace fs = std::filesystem;
+
+int scylla_local_file_key_generator_main(int argc, char** argv) {
+    constexpr auto description_template =
+R"(scylla-{} - a command-line tool to generate file-based encryption keys.
+
+Usage: scylla {} <op> [--option1] [--option2] ... [key path|key name]
+
+Allows creating symmetric keys for use with scylla encryption at rest 
+local key file provider.
+
+Where <op> can be one of:
+
+{}
+)";
+
+    auto op_str = std::ranges::to<std::string>(operations | std::views::transform([] (const operation& op) { 
+        return fmt::format("* {} - {}\n{}", op.name(), op.summary(), op.description()); 
+    }) | std::views::join_with('\n'));
+    tool_app_template::config app_cfg{
+        .name = app_name,
+        .description = seastar::format(description_template, app_name, app_name, op_str),
+        .operations = std::move(operations),
+        .global_options = &global_options,
+        .global_positional_options = &global_positional_options,
+    };
+    tool_app_template app(std::move(app_cfg));
+
+    return app.run_async(argc, argv, [] (const operation& op, const boost::program_options::variables_map& app_config) {
+        std::vector<std::string> files;
+
+        if (app_config.contains("files")) {
+            files = app_config["files"].as<std::vector<std::string>>();
+        }
+        if (files.size() > 1) {
+            throw std::invalid_argument("Too many arguments");
+        }
+        auto alg = app_config["alg"].as<std::string>();
+        auto mode = app_config["block-mode"].as<std::string>();
+        auto padd = app_config["padding"].as<std::string>();
+        auto len = app_config["length"].as<unsigned>();
+
+        if (!padd.ends_with("Padding")) {
+            padd = padd + "Padding";
+        }
+
+        auto java_sig = fmt::format("{}/{}/{}", alg, mode, padd);
+
+        key_info info {
+            .alg = java_sig, .len = len
+        };
+
+        symmetric_key k(info);
+        auto key = k.key();
+        auto hex = base64_encode(key);
+        auto line = fmt::format("{}:{}:{}", java_sig, len, hex);
+        auto key_name = "system_key"s;
+
+        if (!files.empty()) {
+            fs::path f(files.front());
+            if (fs::is_directory(f)) {
+                f = f / key_name;
+            }
+            if (!fs::exists(f)) {
+                auto p = f.parent_path();
+                if (!p.empty()) {
+                    fs::create_directories(p);
+                }
+            }
+            std::ios_base::openmode mode = std::ios_base::out;
+            if (op.name() == "append") {
+                mode |= std::ios_base::ate|std::ios_base::app;
+            } else {
+                mode |= std::ios_base::trunc;
+            }
+
+            if (!fs::exists(f) || op.name() != "append") {
+                // create once so we can enforce proper
+                // permissions. (neither seastar or c++ io is great here)
+                std::ofstream os(f, mode);
+            }
+
+            fs::permissions(f, fs::perms::owner_read|fs::perms::owner_write);
+            std::ofstream os(f, mode);
+
+            os << line << std::endl;
+        } else {
+            std::cout << line << std::endl;
+        }
+        return 0;
+    });
+}
+
+} // namespace tools

utils/config_file.hh

@@ -53,6 +53,21 @@ public:
 template <typename T>
 extern const config_type config_type_for;
 
+template<>
+extern const config_type config_type_for<uint32_t>;
+
+template<>
+extern const config_type config_type_for<sstring>;
+
+template<>
+extern const config_type config_type_for<bool>;
+
+template<>
+extern const config_type config_type_for<std::unordered_map<sstring, sstring>>;
+
+template<>
+extern const config_type config_type_for<std::unordered_map<sstring, std::unordered_map<sstring, sstring>>>;
+
 class config_file {
     static thread_local unsigned s_shard_id;
     struct any_value {
@@ -271,6 +286,8 @@ public:
     config_file(std::initializer_list<cfg_ref> = {});
     config_file(const config_file&) = delete;
 
+    virtual ~config_file() = default;
+
     void add(cfg_ref, std::unique_ptr<any_value> value);
     void add(std::initializer_list<cfg_ref>);
     void add(const std::vector<cfg_ref> &);