From 723518c3902be15224d8fa3cd7fdf253517e9f45 Mon Sep 17 00:00:00 2001 From: Calle Wilund Date: Wed, 8 Jan 2025 10:49:08 +0000 Subject: [PATCH] EAR: port the ear feature from enterprise Bulk transfer of EAR functionality. Includes all providers etc. Could maybe break up into smaller blocks, but once it gets down to the core of it, would require messing with code instead of just moving. So this is it. Note: KMIP support is disabled unless you happen to have the kmipc SDK in your scylla dir. Adds optional encryption of sstables and commitlog, using block level file encryption. Provides key sourcing from various sources, such as local files or popular KMS systems. --- CMakeLists.txt | 2 + cmake/Findkmip.cmake | 53 + configure.py | 41 +- ent/CMakeLists.txt | 1 + ent/encryption/CMakeLists.txt | 44 + ent/encryption/encrypted_file_impl.cc | 555 ++++++++++ ent/encryption/encrypted_file_impl.hh | 24 + ent/encryption/encryption.cc | 1040 ++++++++++++++++++ ent/encryption/encryption.hh | 196 ++++ ent/encryption/encryption_config.cc | 164 +++ ent/encryption/encryption_config.hh | 33 + ent/encryption/encryption_exceptions.hh | 55 + ent/encryption/gcp_host.cc | 1031 +++++++++++++++++ ent/encryption/gcp_host.hh | 80 ++ ent/encryption/gcp_key_provider.cc | 77 ++ ent/encryption/gcp_key_provider.hh | 25 + ent/encryption/kmip_host.cc | 1222 +++++++++++++++++++++ ent/encryption/kmip_host.hh | 80 ++ ent/encryption/kmip_key_provider.cc | 119 ++ ent/encryption/kmip_key_provider.hh | 40 + ent/encryption/kms_host.cc | 1164 ++++++++++++++++++++ ent/encryption/kms_host.hh | 80 ++ ent/encryption/kms_key_provider.cc | 71 ++ ent/encryption/kms_key_provider.hh | 37 + ent/encryption/local_file_provider.cc | 292 +++++ ent/encryption/local_file_provider.hh | 41 + ent/encryption/replicated_key_provider.cc | 477 ++++++++ ent/encryption/replicated_key_provider.hh | 39 + ent/encryption/symmetric_key.cc | 396 +++++++ ent/encryption/symmetric_key.hh | 154 +++ ent/encryption/system_key.cc | 65 ++ ent/encryption/system_key.hh | 34 + 32 files changed, 7731 insertions(+), 1 deletion(-) create mode 100644 cmake/Findkmip.cmake create mode 100644 ent/CMakeLists.txt create mode 100644 ent/encryption/CMakeLists.txt create mode 100644 ent/encryption/encrypted_file_impl.cc create mode 100644 ent/encryption/encrypted_file_impl.hh create mode 100644 ent/encryption/encryption.cc create mode 100644 ent/encryption/encryption.hh create mode 100644 ent/encryption/encryption_config.cc create mode 100644 ent/encryption/encryption_config.hh create mode 100644 ent/encryption/encryption_exceptions.hh create mode 100644 ent/encryption/gcp_host.cc create mode 100644 ent/encryption/gcp_host.hh create mode 100644 ent/encryption/gcp_key_provider.cc create mode 100644 ent/encryption/gcp_key_provider.hh create mode 100644 ent/encryption/kmip_host.cc create mode 100644 ent/encryption/kmip_host.hh create mode 100644 ent/encryption/kmip_key_provider.cc create mode 100644 ent/encryption/kmip_key_provider.hh create mode 100644 ent/encryption/kms_host.cc create mode 100644 ent/encryption/kms_host.hh create mode 100644 ent/encryption/kms_key_provider.cc create mode 100644 ent/encryption/kms_key_provider.hh create mode 100644 ent/encryption/local_file_provider.cc create mode 100644 ent/encryption/local_file_provider.hh create mode 100644 ent/encryption/replicated_key_provider.cc create mode 100644 ent/encryption/replicated_key_provider.hh create mode 100644 ent/encryption/symmetric_key.cc create mode 100644 ent/encryption/symmetric_key.hh create mode 100644 ent/encryption/system_key.cc create mode 100644 ent/encryption/system_key.hh diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a19edda5e..cd32630f91 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -267,6 +267,7 @@ add_subdirectory(compaction) add_subdirectory(cql3) add_subdirectory(data_dictionary) add_subdirectory(dht) +add_subdirectory(ent) add_subdirectory(gms) add_subdirectory(idl) add_subdirectory(index) @@ -308,6 +309,7 @@ set(scylla_libs cql3 data_dictionary dht + encryption gms idl index diff --git a/cmake/Findkmip.cmake b/cmake/Findkmip.cmake new file mode 100644 index 0000000000..02b9169711 --- /dev/null +++ b/cmake/Findkmip.cmake @@ -0,0 +1,53 @@ +# +# Copyright 2024-present ScyllaDB +# + +# +# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 +# + +set(kmip_ver "2.1.0t") + +cmake_host_system_information( + RESULT distrib_id QUERY DISTRIB_ID) +if(distrib_id MATCHES "centos|fedora|rhel") + set(kmip_distrib "rhel84") +else() + message(FATAL_ERROR "Could not locate kmipc library for ${distrib_id}") +endif() + +if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64") + set(kmip_arch "aarch64") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64") + set(kmip_arch "64") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(powerpc|ppc)64le") + set(kmip_arch "ppc64le") +endif() + +set(kmip_ROOT "${PROJECT_SOURCE_DIR}/kmipc/kmipc-${kmip_ver}-${kmip_distrib}_${kmip_arch}") +find_library(kmip_LIBRARY + NAMES kmip + HINTS ${kmip_ROOT}/lib) + +find_path(kmip_INCLUDE_DIR + NAMES kmip.h + HINTS ${kmip_ROOT}/include) + +mark_as_advanced( + kmip_LIBRARY + kmip_INCLUDE_DIR) + +find_package_handle_standard_args(kmip + REQUIRED_VARS + kmip_LIBRARY + kmip_INCLUDE_DIR) + +if(kmip_FOUND) + if (NOT TARGET KMIP::kmipc) + add_library(KMIP::kmipc UNKNOWN IMPORTED) + set_target_properties(KMIP::kmipc PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${kmip_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${kmip_LIBRARY}") + endif() +endif() diff --git a/configure.py b/configure.py index 03fe64ba8e..5e930ff6db 100755 --- a/configure.py +++ b/configure.py @@ -1123,6 +1123,19 @@ scylla_core = (['message/messaging_service.cc', 'utils/arch/powerpc/crc32-vpmsum/crc32_wrapper.cc', 'querier.cc', 'mutation_writer/multishard_writer.cc', + 'ent/encryption/encryption_config.cc', + 'ent/encryption/encryption.cc', + 'ent/encryption/symmetric_key.cc', + 'ent/encryption/local_file_provider.cc', + 'ent/encryption/replicated_key_provider.cc', + 'ent/encryption/system_key.cc', + 'ent/encryption/encrypted_file_impl.cc', + 'ent/encryption/kmip_host.cc', + 'ent/encryption/kmip_key_provider.cc', + 'ent/encryption/kms_host.cc', + 'ent/encryption/kms_key_provider.cc', + 'ent/encryption/gcp_host.cc', + 'ent/encryption/gcp_key_provider.cc', 'multishard_mutation_query.cc', 'reader_concurrency_semaphore.cc', 'sstables_loader.cc', @@ -2000,7 +2013,7 @@ pkgs = ['libsystemd', pkgs.append('lua53' if have_pkg('lua53') else 'lua') -libs = ' '.join([maybe_static(args.staticyamlcpp, '-lyaml-cpp'), '-latomic', '-lz', '-lsnappy', +libs = ' '.join([maybe_static(args.staticyamlcpp, '-lyaml-cpp'), '-latomic', '-lz', '-lsnappy', '-lcrypto', ' -lstdc++fs', ' -lcrypt', ' -lcryptopp', ' -lpthread', # Must link with static version of libzstd, since # experimental APIs that we use are only present there. @@ -2022,6 +2035,32 @@ user_ldflags += ' -fvisibility=hidden' if args.staticcxx: user_ldflags += " -static-libstdc++" +kmip_lib_ver = '1.9.2a'; + +def kmiplib(): + os_ids = get_os_ids() + for id in os_ids: + if id in { 'centos', 'fedora', 'rhel' }: + return 'rhel84' + print('Could not resolve libkmip.a for platform {}'.format(os_ids)) + sys.exit(1) + +def target_cpu(): + cpu, _, _ = subprocess.check_output([cxx, '-dumpmachine']).decode('utf-8').partition('-') + return cpu + +def kmip_arch(): + arch = target_cpu() + if arch == 'x86_64': + return '64' + return arch + +kmipc_dir = f'kmipc/kmipc-2.1.0t-{kmiplib()}_{kmip_arch()}' +kmipc_lib = f'{kmipc_dir}/lib/libkmip.a' +libs += ' -lboost_filesystem' +if os.path.exists(kmipc_lib): + libs += f' {kmipc_lib}' + user_cflags += f' -I{kmipc_dir}/include -DHAVE_KMIP' def get_extra_cxxflags(mode, mode_config, cxx, debuginfo): cxxflags = [] diff --git a/ent/CMakeLists.txt b/ent/CMakeLists.txt new file mode 100644 index 0000000000..7ca6eb6617 --- /dev/null +++ b/ent/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(encryption) diff --git a/ent/encryption/CMakeLists.txt b/ent/encryption/CMakeLists.txt new file mode 100644 index 0000000000..2bee22124b --- /dev/null +++ b/ent/encryption/CMakeLists.txt @@ -0,0 +1,44 @@ +include(add_whole_archive) + +find_package(cpp-jwt REQUIRED) +find_package(kmip) + +add_library(scylla_encryption STATIC) +target_sources(scylla_encryption + PRIVATE + encrypted_file_impl.cc + encryption.cc + encryption_config.cc + gcp_host.cc + gcp_key_provider.cc + kmip_host.cc + kmip_key_provider.cc + kms_host.cc + kms_key_provider.cc + local_file_provider.cc + replicated_key_provider.cc + symmetric_key.cc + system_key.cc) +target_include_directories(scylla_encryption + PUBLIC + ${CMAKE_SOURCE_DIR}) +target_link_libraries(scylla_encryption + PUBLIC + Seastar::seastar + PRIVATE + cql3 + utils + cpp-jwt::cpp-jwt) +if(kmip_FOUND) + target_link_libraries(scylla_encryption + PRIVATE + KMIP::kmipc) + target_compile_definitions(scylla_encryption + PUBLIC + HAVE_KMIP) +endif() + +check_headers(check-headers scylla_encryption + GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh) + +add_whole_archive(encryption scylla_encryption) diff --git a/ent/encryption/encrypted_file_impl.cc b/ent/encryption/encrypted_file_impl.cc new file mode 100644 index 0000000000..7c3cf86d49 --- /dev/null +++ b/ent/encryption/encrypted_file_impl.cc @@ -0,0 +1,555 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#include +#include +#include +#include + +#include "symmetric_key.hh" +#include "encryption.hh" +#include "utils/serialization.hh" +#include "encrypted_file_impl.hh" + +namespace encryption { + +static inline bool is_aligned(size_t n, size_t a) { + return (n & (a - 1)) == 0; +} + +/** + * Very simple block-encrypting file wrapper. + * + * Uses user provided symmetric key + ESSIV block IV calculation + * to encrypt data. + * + * The essiv block key is created by generating the SHA256 hash + * of the provided data encryption key bytes, truncated to block_key_len/8 + * and generating an AES/ECB key using this data. + * + * The file is divided in N blocks of `block_size` size. + * Each block is encrypted (unpadded) with the provided key and + * block mode, using an IV derived by (essiv): + * + * bytes tmp[] = { 0, ..., uint64_t-little-endian() } + * iv = block_key->encrypt(tmp); + * + * All encryption is done unpadded. To handle file sizes we use + * a slightly shaky scheme: + * + * Since all writes are assumed to be done by us, and must be aligned, + * we can assume in turn that any resizing should be made by our truncate + * method. If we attept to truncate to a size not a multiple of our + * _key_ block size (typically 16), we add the same size to the actual + * truncation size. + * On read we then check the file size. If we're reading from a file + * with unaliged size, we know there are key-block-size junk at the end. + * We can align down the last decryption call to match block size, then + * discard the excessive bytes from the result. + * + * If we're in a read/write situation, we need to keep size updated, and + * we could possibly race with disk op/continuations. + * But we are really only for ro/wo cases. + */ +class encrypted_file_impl : public seastar::file_impl { + file _file; + ::shared_ptr _key; + ::shared_ptr _block_key; + bytes _hash_salt; + + std::optional _file_length; + + // this is somewhat large, but we assume this is for bulky stuff like sstables/commitlog + // so large alignment should be preferable to reaclculating block IV too often. + static constexpr size_t block_size = 4096; + static constexpr size_t block_key_len = 256; + + class my_file_handle_impl; + friend class my_file_handle_impl; + + bytes iv_for(uint64_t pos) const; + + using mode = symmetric_key::mode; + + temporary_buffer transform(uint64_t, const void* buffer, size_t len, mode); + size_t transform(uint64_t, const void* buffer, size_t len, void*, mode); + + future<> verify_file_length(); + void maybe_set_length(uint64_t); + void clear_length(); + + static ::shared_ptr generate_block_key(::shared_ptr); + +public: + encrypted_file_impl(file, ::shared_ptr); + + future write_dma(uint64_t pos, const void* buffer, size_t len, io_intent*) override; + future write_dma(uint64_t pos, std::vector iov, io_intent*) override; + future read_dma(uint64_t pos, void* buffer, size_t len, io_intent*) override; + future read_dma(uint64_t pos, std::vector iov, io_intent*) override; + + + future<> flush() override { + return _file.flush(); + } + future stat(void) override; + future<> truncate(uint64_t length) override; + future<> discard(uint64_t offset, uint64_t length) override { + return _file.discard(offset, length); + } + future<> allocate(uint64_t position, uint64_t length) override { + return _file.allocate(position, length); + } + future size(void) override; + future<> close() override { + return _file.close(); + } + std::unique_ptr dup() override; + + subscription list_directory(std::function (directory_entry de)> next) override { + return _file.list_directory(std::move(next)); + } + future> dma_read_bulk(uint64_t offset, size_t range_size, io_intent*) override; +}; + +/** + * Note: ESSIV block iv generation implementation. + * See: http://securityevaluators.com/knowledge/papers/fde_whitepaper_draft_20170627.pdf + * + * We generate a key based on the sha256 of the data key, then encrypt each block number + * using this to get per-block IV. + * The key is AES-256, using ECB (non-iv) encryption + * + */ +::shared_ptr encrypted_file_impl::generate_block_key(::shared_ptr key) { + auto hash = calculate_sha256(key->key()); + hash.resize(block_key_len / 8); + return ::make_shared(key_info{"AES/ECB", block_key_len }, hash); +} + +encrypted_file_impl::encrypted_file_impl(file f, ::shared_ptr key) + : _file(std::move(f)) + , _key(std::move(key)) + , _block_key(generate_block_key(_key)) +{ + _memory_dma_alignment = std::max(_file.memory_dma_alignment(), block_size); + _disk_read_dma_alignment = std::max(_file.disk_read_dma_alignment(), block_size); + _disk_write_dma_alignment = std::max(_file.disk_write_dma_alignment(), block_size); +} + +static future calculate_file_length(const file& f, size_t key_block_size) { + return f.size().then([key_block_size](uint64_t s) { + if (!is_aligned(s, key_block_size)) { + if (s < key_block_size) { + throw std::domain_error(fmt::format("file size {}, expected 0 or at least {}", s, key_block_size)); + } + s -= key_block_size; + } + return s; + }); +} + +future<> encrypted_file_impl::verify_file_length() { + if (_file_length) { + return make_ready_future(); + } + return calculate_file_length(_file, _key->block_size()).then([this](uint64_t s) { + _file_length = s; + }); +} + +void encrypted_file_impl::maybe_set_length(uint64_t s) { + if (s > _file_length.value_or(0)) { + _file_length = s; + } +} + +void encrypted_file_impl::clear_length() { + _file_length = std::nullopt; +} + +bytes encrypted_file_impl::iv_for(uint64_t pos) const { + assert(!(pos & (block_size - 1))); + + // #658. ECB block mode has no IV. Bad for security, + // but must handle. + size_t iv_len = _key->iv_len(); + if (iv_len == 0) { + return bytes{}; + } + + assert(iv_len >= _key->block_size()); + assert(iv_len >= sizeof(uint64_t)); + + bytes b(bytes::initialized_later(), std::max(iv_len, _block_key->block_size())); + std::fill(b.begin(), b.end() - sizeof(uint64_t), 0); + + // write block pos as little endian IV-len integer + auto block = pos / block_size; + write_le(reinterpret_cast(b.end()) - sizeof(uint64_t), block); + + // encrypt the encoded block number to build an IV + _block_key->encrypt_unpadded(b.data(), b.size(), b.data()); + + b.resize(iv_len); + + return b; +} + +size_t encrypted_file_impl::transform(uint64_t pos, const void* buffer, size_t len, void* dst, mode m) { + assert(!(pos & (block_size - 1))); + assert(_file_length || m == mode::encrypt); + + auto o = reinterpret_cast(dst); + auto i = reinterpret_cast(buffer); + auto l = _file_length.value_or(std::numeric_limits::max()); + auto b = _key->block_size(); + + size_t off = 0; + + for (; off < len; off += block_size) { + auto iv = iv_for(pos + off); + auto rem = std::min(block_size, len - off); + + if (rem < block_size || ((pos + off + rem) > l && m == symmetric_key::mode::decrypt)) { + // truncated block. should be the last one. + if (m != symmetric_key::mode::decrypt) { + throw std::invalid_argument("Output data not aligned"); + } + _key->transform_unpadded(m, i + off, align_down(rem, b), o + off, iv.data()); + return l - pos; + } + _key->transform_unpadded(m, i + off, block_size, o + off, iv.data()); + } + + return off; +} + +temporary_buffer encrypted_file_impl::transform(uint64_t pos, const void* buffer, size_t len, mode m) { + assert(!(len & (block_size - 1))); + auto tmp = temporary_buffer::aligned(_file.memory_dma_alignment(), len); + auto s = transform(pos, buffer, len, tmp.get_write(), m); + tmp.trim(s); + return tmp; +} + +future encrypted_file_impl::write_dma(uint64_t pos, const void* buffer, size_t len, io_intent* intent) { + assert(!(len & (block_size - 1))); + auto tmp = transform(pos, buffer, len, mode::encrypt); + assert(tmp.size() == len); // writing + auto p = tmp.get(); + return _file.dma_write(pos, p, len, intent).then([this, tmp = std::move(tmp), pos](size_t s) { + maybe_set_length(pos + s); + return s; + }); +} + +future encrypted_file_impl::write_dma(uint64_t pos, std::vector iov, io_intent* intent) { + std::vector> tmp; + tmp.reserve(iov.size()); + size_t n = 0; + for (auto& i : iov) { + assert(!(i.iov_len & (block_size - 1))); + + tmp.emplace_back(transform(pos + n, i.iov_base, i.iov_len, mode::encrypt)); + assert(tmp.back().size() == i.iov_len); // writing + n += i.iov_len; + i = iovec{ tmp.back().get_write(), tmp.back().size() }; + } + return _file.dma_write(pos, std::move(iov), intent).then([this, tmp = std::move(tmp), pos](size_t s) { + maybe_set_length(pos + s); + return s; + }); +} + +future encrypted_file_impl::read_dma(uint64_t pos, void* buffer, size_t len, io_intent* intent) { + assert(!(len & (block_size - 1))); + return verify_file_length().then([this, pos, buffer, len, intent] { + return _file.dma_read(pos, buffer, len, intent).then([this, pos, buffer](size_t len) { + return transform(pos, buffer, len, buffer, mode::decrypt); + }); + }); +} + +future encrypted_file_impl::read_dma(uint64_t pos, std::vector iov, io_intent* intent) { + return verify_file_length().then([this, pos, iov = std::move(iov), intent]() mutable { + auto f = _file.dma_read(pos, iov, intent); + return f.then([this, pos, iov = std::move(iov)](size_t len) mutable { + size_t off = 0; + for (auto& i : iov) { + off += transform(pos + off, i.iov_base, i.iov_len, i.iov_base, mode::decrypt); + } + return off; + }); + }); +} + +future> encrypted_file_impl::dma_read_bulk(uint64_t offset, size_t range_size, io_intent* intent) { + return verify_file_length().then([this, offset, range_size, intent]() mutable { + auto front = offset & (block_size - 1); + offset -= front; + range_size += front; + // enterprise #925 + // If caller is clever and asks for the last chunk of file + // explicitly (as in offset = N, range_size = size() - N), + // or any other unaligned size, we need to add enough padding + // to get the actual full block to decode. + auto block_size = align_up(range_size, _key->block_size()); + return _file.dma_read_bulk(offset, block_size, intent).then([this, offset, front, range_size](temporary_buffer result) { + auto s = transform(offset, result.get(), result.size(), result.get_write(), mode::decrypt); + // never give back more than asked for. + result.trim(std::min(s, range_size)); + result.trim_front(front); + return result; + }); + }); +} + +future<> encrypted_file_impl::truncate(uint64_t length) { + return size().then([this, length](uint64_t s) { + if (s >= length) { + auto kb = _key->block_size(); + auto n = length; + if (!is_aligned(length, kb)) { + n += kb; + } + return _file.truncate(n).then([this, length] { + _file_length = length; + }); + } + + // crap. we need to pad zeros. But zeros here means + // encrypted zeros. So we must do this surprisingly + // expensively, by actually writing said zeros block + // by block. Anyone hoping for sparse files is now + // severely disappointed! + + auto buf_size = align_up(std::min(length, 32 * block_size), block_size); + auto aligned_size = align_down(s, block_size); + + temporary_buffer buf(buf_size); + std::fill(buf.get_write(), buf.get_write() + buf_size, 0); + + struct trunc { + temporary_buffer buf; + uint64_t aligned_size; + uint64_t size; + uint64_t length; + }; + + return do_with(trunc{std::move(buf), aligned_size, s, length}, [this](trunc & t) { + return repeat([this, &t] { + if (t.aligned_size >= t.length) { + return make_ready_future(stop_iteration::yes); + } + auto n = std::min(t.buf.size(), align_up(size_t(t.length - t.aligned_size), block_size)); + if (t.aligned_size < t.size) { + return read_dma(t.aligned_size, t.buf.get_write(), n, nullptr).then([&, n](size_t r) mutable { + auto rem = size_t(t.size - t.aligned_size); + auto ar = align_up(r, block_size); + assert(ar <= t.buf.size()); + if (rem < ar) { + std::fill(t.buf.get_write() + rem, t.buf.get_write() + ar, 0); + } + return write_dma(t.aligned_size, t.buf.get(), ar, nullptr).then([&, n](size_t w) { + t.aligned_size += w; + // #1869. On btrfs, we get the buffer potentially clobbered up to "n" (max read amount) + // even when "r" (actual bytes read) is less. + std::fill(t.buf.get_write(), t.buf.get_write() + n, 0); + return make_ready_future(stop_iteration::no); + }); + }); + } + return write_dma(t.aligned_size, t.buf.get(), n, nullptr).then([&](size_t w) { + t.aligned_size += w; + return make_ready_future(stop_iteration::no); + }); + }); + }).then([this, length] { + return truncate(length); + });; + }); +} + +future encrypted_file_impl::stat() { + return _file.stat().then([this](struct stat s) { + return verify_file_length().then([this, s]() mutable { + s.st_size = *_file_length; + return s; + }); + }); +} + +future encrypted_file_impl::size() { + return verify_file_length().then([this] { + return *_file_length; + }); +} + + +std::unique_ptr encrypted_file_impl::dup() { + class my_file_handle_impl : public seastar::file_handle_impl { + seastar::file_handle _handle; + key_info _info; + bytes _key; + public: + my_file_handle_impl(seastar::file_handle h, const key_info& info, const bytes& key) + : _handle(std::move(h)) + , _info(info) + , _key(key) + {} + std::unique_ptr clone() const override { + return std::make_unique(_handle, _info, _key); + } + seastar::shared_ptr to_file() && override { + return seastar::make_shared(_handle.to_file(), ::make_shared(_info, _key)); + } + }; + + return std::make_unique(_file.dup(), _key->info(), _key->key()); +} + +shared_ptr make_encrypted_file(file f, ::shared_ptr k) { + return ::make_shared(std::move(f), std::move(k)); +} + +class indirect_encrypted_file_impl : public file_impl { + ::shared_ptr _impl; + file _f; + size_t _key_block_size; + get_key_func _get; + + future<> get() { + if (_impl) { + return make_ready_future<>(); + } + return _get().then([this](::shared_ptr k) { + // #978 could be running the getting more than once. + // Only write _impl once though + if (!_impl) { + _impl = make_encrypted_file(_f, std::move(k)); + } + }); + } +public: + indirect_encrypted_file_impl(file f, size_t key_block_size, get_key_func get) + : _f(f), _key_block_size(key_block_size), _get(std::move(get)) + {} + + future write_dma(uint64_t pos, const void* buffer, size_t len, io_intent* intent) override { + return get().then([this, pos, buffer, len, intent]() { + return _impl->write_dma(pos, buffer, len, intent); + }); + } + future write_dma(uint64_t pos, std::vector iov, io_intent* intent) override { + return get().then([this, pos, iov = std::move(iov), intent]() mutable { + return _impl->write_dma(pos, std::move(iov), intent); + }); + } + future read_dma(uint64_t pos, void* buffer, size_t len, io_intent* intent) override { + return get().then([this, pos, buffer, len, intent]() { + return _impl->read_dma(pos, buffer, len, intent); + }); + } + future read_dma(uint64_t pos, std::vector iov, io_intent* intent) override { + return get().then([this, pos, iov = std::move(iov), intent]() mutable { + return _impl->read_dma(pos, std::move(iov), intent); + }); + } + future> dma_read_bulk(uint64_t offset, size_t range_size, io_intent* intent) override { + return get().then([this, offset, range_size, intent]() { + return _impl->dma_read_bulk(offset, range_size, intent); + }); + } + future<> flush(void) override { + if (_impl) { + return _impl->flush(); + } + return _f.flush(); + } + future stat(void) override { + if (_impl) { + return _impl->stat(); + } + return _f.stat().then([this](struct stat s) { + return calculate_file_length(_f, _key_block_size).then([s](uint64_t fs) mutable { + s.st_size = fs; + return s; + }); + }); + } + future<> truncate(uint64_t length) override { + if (_impl) { + return _impl->truncate(length); + } + return _f.truncate(length); + } + future<> discard(uint64_t offset, uint64_t length) override { + if (_impl) { + return _impl->discard(offset, length); + } + return _f.discard(offset, length); + } + future<> allocate(uint64_t position, uint64_t length) override { + if (_impl) { + return _impl->allocate(position, length); + } + return _f.allocate(position, length); + } + future size(void) override { + if (_impl) { + return _impl->size(); + } + return calculate_file_length(_f, _key_block_size); + } + future<> close() override { + if (_impl) { + return _impl->close(); + } + return _f.close(); + } + std::unique_ptr dup() override { + if (_impl) { + return _impl->dup(); + } + class my_file_handle_impl : public seastar::file_handle_impl { + seastar::file_handle _handle; + size_t _key_block_size; + get_key_func _get; + public: + my_file_handle_impl(seastar::file_handle h, size_t key_block_size, get_key_func get) + : _handle(std::move(h)) + , _key_block_size(key_block_size) + , _get(std::move(get)) + {} + std::unique_ptr clone() const override { + return std::make_unique(_handle, _key_block_size, _get); + } + seastar::shared_ptr to_file() && override { + return make_delayed_encrypted_file(_handle.to_file(), _key_block_size, _get); + } + }; + return std::make_unique(_f.dup(), _key_block_size, _get); + } + + subscription list_directory(std::function (directory_entry de)> next) override { + if (_impl) { + return _impl->list_directory(std::move(next)); + } + return _f.list_directory(std::move(next)); + } +}; + +shared_ptr make_delayed_encrypted_file(file f, size_t key_block_size, get_key_func get) { + return ::make_shared(std::move(f), key_block_size, std::move(get)); +} + + +} + diff --git a/ent/encryption/encrypted_file_impl.hh b/ent/encryption/encrypted_file_impl.hh new file mode 100644 index 0000000000..f4ecc7719b --- /dev/null +++ b/ent/encryption/encrypted_file_impl.hh @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#include +#include + +#include "symmetric_key.hh" + +namespace encryption { + +class symmetric_key; + +shared_ptr make_encrypted_file(file, ::shared_ptr); + +using get_key_func = std::function>()>; + +shared_ptr make_delayed_encrypted_file(file, size_t, get_key_func); +} diff --git a/ent/encryption/encryption.cc b/ent/encryption/encryption.cc new file mode 100644 index 0000000000..0031b06843 --- /dev/null +++ b/ent/encryption/encryption.cc @@ -0,0 +1,1040 @@ +/* + * Copyright (C) 2015 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include "utils/to_string.hh" + +#include "compress.hh" +#include "encryption.hh" +#include "symmetric_key.hh" +#include "local_file_provider.hh" +#include "replicated_key_provider.hh" +#include "kmip_key_provider.hh" +#include "kmip_host.hh" +#include "kms_key_provider.hh" +#include "kms_host.hh" +#include "gcp_key_provider.hh" +#include "gcp_host.hh" +#include "bytes.hh" +#include "utils/class_registrator.hh" +#include "cql3/query_processor.hh" +#include "db/extensions.hh" +#include "db/system_keyspace.hh" +#include "serializer.hh" +#include "serializer_impl.hh" +#include "schema/schema.hh" +#include "sstables/sstables.hh" +#include "service/storage_service.hh" +#include "service/migration_manager.hh" +#include "db/commitlog/commitlog_extensions.hh" +#include "encrypted_file_impl.hh" +#include "encryption_config.hh" +#include "utils/UUID_gen.hh" +#include "init.hh" + +static seastar::logger logg{"encryption"}; + +namespace encryption { + +static constexpr auto REPLICATED_KEY_PROVIDER_FACTORY = "ReplicatedKeyProviderFactory"; +static constexpr auto LOCAL_FILE_SYSTEM_KEY_PROVIDER_FACTORY = "LocalFileSystemKeyProviderFactory"; +static constexpr auto KMIP_KEY_PROVIDER_FACTORY = "KmipKeyProviderFactory"; +static constexpr auto KMS_KEY_PROVIDER_FACTORY = "KmsKeyProviderFactory"; +static constexpr auto GCP_KEY_PROVIDER_FACTORY = "GcpKeyProviderFactory"; + +bytes base64_decode(const sstring& s, size_t off, size_t len) { + if (off >= s.size()) { + throw std::out_of_range("Invalid offset"); + } + len = std::min(len, s.size() - off); + auto n = (len / 4) * 3; + bytes b{bytes::initialized_later(), n}; + + // EVP_DecodeBlock does not handle padding well (i.e. it returns + // data with actual padding. This is not what we want, since + // we need to allow zeros in data. + // Must thus do decoding the hard way... + + std::unique_ptr ctxt(EVP_ENCODE_CTX_new(), &EVP_ENCODE_CTX_free); + + ::EVP_DecodeInit(ctxt.get()); + + int outl = 0; + auto r = ::EVP_DecodeUpdate(ctxt.get(), reinterpret_cast(b.data()), &outl, reinterpret_cast(s.data() + off), + int(len)); + if (r < 0) { + throw std::invalid_argument("Could not decode: " + s); + } + + int outl2 = 0; + r = ::EVP_DecodeFinal(ctxt.get(), reinterpret_cast(b.data() + outl), &outl2); + if (r < 0) { + throw std::invalid_argument("Could not decode: " + s); + } + b.resize(outl + outl2); + return b; +} + +sstring base64_encode(const bytes& b, size_t off, size_t len) { + if (off >= b.size()) { + throw std::out_of_range("Invalid offset"); + } + len = std::min(len, b.size() - off); + auto n = ((len + 2) / 3) * 4; + sstring s{sstring::initialized_later(), n}; + auto r = EVP_EncodeBlock(reinterpret_cast(s.data()), + reinterpret_cast(b.data() + off), int(len)); + if (r < 0) { + throw std::invalid_argument("Could not encode"); + } + s.resize(r); + return s; +} + +bytes calculate_md5(const bytes& b, size_t off, size_t len) { + if (off >= b.size()) { + throw std::out_of_range("Invalid offset"); + } + len = std::min(len, b.size() - off); + bytes res{bytes::initialized_later(), MD5_DIGEST_LENGTH}; +#if OPENSSL_VERSION_NUMBER >= (3<<28) + EVP_MD_CTX *md5 = EVP_MD_CTX_new(); + EVP_DigestInit_ex(md5, EVP_md5(), nullptr); + EVP_DigestUpdate(md5, b.data() + off, len); + EVP_DigestFinal_ex(md5, reinterpret_cast(res.data()), nullptr); + EVP_MD_CTX_free(md5); +#else + MD5(reinterpret_cast(b.data() + off), len, reinterpret_cast(res.data())); +#endif + return res; +} + +bytes calculate_sha256(bytes_view b) { + bytes res{bytes::initialized_later(), SHA256_DIGEST_LENGTH}; + SHA256(reinterpret_cast(b.data()), b.size(), reinterpret_cast(res.data())); + return res; +} + +bytes calculate_sha256(const bytes& b, size_t off, size_t len) { + if (off >= b.size()) { + throw std::out_of_range("Invalid offset"); + } + len = std::min(len, b.size() - off); + return calculate_sha256(bytes_view(b.data() + off, len)); +} + +bytes hmac_sha256(bytes_view msg, bytes_view key) { + bytes res{bytes::initialized_later(), SHA256_DIGEST_LENGTH}; + + unsigned length; + HMAC(EVP_sha256(), + key.data(), key.size(), + reinterpret_cast(msg.data()), msg.size(), + reinterpret_cast(res.data()), &length); + return res; +} + +future> read_text_file_fully(const sstring& filename) { + return open_file_dma(filename, open_flags::ro).then([](file f) { + return f.size().then([f](size_t s) { + return do_with(make_file_input_stream(f), [s](input_stream& in) { + return in.read_exactly(s).then([](temporary_buffer buf) { + return make_ready_future>(std::move(buf)); + }).finally([&in] { + return in.close(); + }); + }); + }); + }); +} + +future<> write_text_file_fully(const sstring& filename, temporary_buffer buf) { + return open_file_dma(filename, open_flags::wo|open_flags::create).then([buf = std::move(buf)](file f) mutable { + return make_file_output_stream(f).then([buf = std::move(buf)] (output_stream out) mutable { + return do_with(std::move(out), [buf = std::move(buf)](output_stream& out) mutable { + auto p = buf.get(); + auto s = buf.size(); + return out.write(p, s).finally([&out, buf = std::move(buf)] { + return out.close(); + }); + }); + }); + }); +} + +future<> write_text_file_fully(const sstring& filename, const sstring& s) { + return write_text_file_fully(filename, temporary_buffer(s.data(), s.size())); +} + +std::optional parse_expiry(std::optional in) { + if (!in) { + return std::nullopt; + } + size_t idx = 0; + auto n = std::stoll(*in, &idx); // we assume seconds + + if (idx != 0) { + auto unit = in->substr(idx); + if (unit == "ms") { + return std::chrono::milliseconds(n); + } else if (unit == "h") { + return std::chrono::duration_cast(std::chrono::hours(n)); + } else if (unit == "d") { + return std::chrono::duration_cast(std::chrono::days(n)); + } else if (unit == "s") { + // ok + } else if (unit != "") { + throw std::invalid_argument("Unsupported time unit: " + unit); + } + } + return std::chrono::duration_cast(std::chrono::seconds(n)); +} + + +static const sstring namespace_prefix = "com.datastax.bdp.cassandra.crypto."; +static const sstring encryption_attribute = "scylla_encryption_options"; + +static inline const sstring key_id_attribute = "scylla_key_id"; +static inline const sstring encrypted_components_attribute = "encrypted_components"; + +static inline const sstables::disk_string encryption_attribute_ds{ + bytes{encryption_attribute.begin(), encryption_attribute.end()} +}; +static inline const sstables::disk_string key_id_attribute_ds{ + bytes{key_id_attribute.begin(), key_id_attribute.end()} +}; +static inline const sstables::disk_string encrypted_components_attribute_ds{ + bytes{encrypted_components_attribute.begin(), encrypted_components_attribute.end()} +}; + +key_info get_key_info(const options& map) { + opt_wrapper opts(map); + + auto cipher_name = opts(CIPHER_ALGORITHM).value_or("AES/CBC/PKCS5Padding"); + auto key_strength = std::stoul(opts(SECRET_KEY_STRENGTH).value_or("128")); + // todo: static constexpr auto KMIP_KEY_PROVIDER_FACTORY = "KmipKeyProviderFactory"; + return key_info{ std::move(cipher_name), unsigned(key_strength) }; +} + +std::ostream& operator<<(std::ostream& os, const key_provider& p) { + p.print(os); + return os; +} + +sstring encryption_context::maybe_decrypt_config_value(const sstring& s) const { + shared_ptr k = get_config_encryption_key(); + if (!s.empty() && k != nullptr) { + auto b = base64_decode(s); + auto iv = calculate_sha256(k->key()); + iv.resize(k->block_size(), 0); + bytes dst(bytes::initialized_later(), b.size()); + auto len = k->decrypt(b.data(), b.size(), dst.data(), dst.size(), iv.data()); + return sstring(dst.begin(), dst.begin() + len); + } + return s; +} + +class encryption_schema_extension; + +class encryption_context_impl : public encryption_context { + // poor mans per-thread instance variable. We need a lookup map + // per shard, so preallocate it, much like a "sharded" thing would, + // but without all the fancy start/stop stuff. + // Allows this object to be effectively stateless, except for the + // objects in the maps. + std::vector>> _per_thread_provider_cache; + std::vector>> _per_thread_system_key_cache; + std::vector>> _per_thread_kmip_host_cache; + std::vector>> _per_thread_kms_host_cache; + std::vector>> _per_thread_gcp_host_cache; + std::vector> _per_thread_global_user_extension; + std::unique_ptr _cfg; + sharded* _qp;; + sharded* _mm; + sharded* _db; + sharded* _ss; + shared_ptr _cfg_encryption_key; + bool _allow_per_table_encryption; +public: + encryption_context_impl(std::unique_ptr cfg, const service_set& services) + : _per_thread_provider_cache(smp::count) + , _per_thread_system_key_cache(smp::count) + , _per_thread_kmip_host_cache(smp::count) + , _per_thread_kms_host_cache(smp::count) + , _per_thread_gcp_host_cache(smp::count) + , _per_thread_global_user_extension(smp::count) + , _cfg(std::move(cfg)) + , _qp(find_or_null(services)) + , _mm(find_or_null(services)) + , _db(find_or_null(services)) + , _ss(find_or_null(services)) + , _allow_per_table_encryption(_cfg->allow_per_table_encryption()) + {} + + template + static sharded* find_or_null(const service_set& services) { + try { + return std::addressof(services.find()); + } catch (std::out_of_range&) { + // TODO: would be great if we could verify we are in tool mode here. + return nullptr; + } + } + + shared_ptr get_provider(const options& map) override { + opt_wrapper opts(map); + + auto provider_class = opts(KEY_PROVIDER); + if (!provider_class) { + provider_class = opts(SECRET_KEY_PROVIDER_FACTORY_CLASS).value_or(REPLICATED_KEY_PROVIDER_FACTORY); + } + if (provider_class->empty() || ::strcasecmp(provider_class->c_str(), "none") == 0) { + return {}; + } + static const std::unordered_map> providers = [] { + std::unordered_map> map; + + map[REPLICATED_KEY_PROVIDER_FACTORY] = std::make_unique(); + map[LOCAL_FILE_SYSTEM_KEY_PROVIDER_FACTORY] = std::make_unique(); + map[KMIP_KEY_PROVIDER_FACTORY] = std::make_unique(); + map[KMS_KEY_PROVIDER_FACTORY] = std::make_unique(); + map[GCP_KEY_PROVIDER_FACTORY] = std::make_unique(); + + return map; + }(); + + unqualified_name qn(namespace_prefix, *provider_class); + + try { + return providers.at(qn)->get_provider(*this, map); + } catch (std::out_of_range&) { + throw std::invalid_argument("Unknown provider: " + *provider_class); + } + } + shared_ptr get_cached_provider(const sstring& id) const override { + auto& cache = _per_thread_provider_cache[this_shard_id()]; + auto i = cache.find(id); + if (i != cache.end()) { + return i->second; + } + return {}; + } + void cache_provider(const sstring& id, shared_ptr p) override { + _per_thread_provider_cache[this_shard_id()][id] = std::move(p); + } + + shared_ptr get_system_key(const sstring& name) override { + auto& cache = _per_thread_system_key_cache[this_shard_id()]; + auto i = cache.find(name); + if (i != cache.end()) { + return i->second; + } + + shared_ptr k; + + if (kmip_system_key::is_kmip_path(name)) { + k = make_shared(*this, name); + } else { + k = make_shared(*this, name); + } + + if (k != nullptr) { + cache[name] = k; + } + + return k; + } + + shared_ptr get_kmip_host(const sstring& host) override { + auto& cache = _per_thread_kmip_host_cache[this_shard_id()]; + auto i = cache.find(host); + if (i != cache.end()) { + return i->second; + } + + auto j = _cfg->kmip_hosts().find(host); + if (j != _cfg->kmip_hosts().end()) { + auto result = ::make_shared(*this, host, j->second); + cache.emplace(host, result); + return result; + } + + throw std::invalid_argument("No such host: "+ host); + } + + shared_ptr get_kms_host(const sstring& host) override { + auto& cache = _per_thread_kms_host_cache[this_shard_id()]; + auto i = cache.find(host); + if (i != cache.end()) { + return i->second; + } + + auto j = _cfg->kms_hosts().find(host); + if (j != _cfg->kms_hosts().end()) { + auto result = ::make_shared(*this, host, j->second); + cache.emplace(host, result); + return result; + } + + throw std::invalid_argument("No such host: "+ host); + } + + shared_ptr get_gcp_host(const sstring& host) override { + auto& cache = _per_thread_gcp_host_cache[this_shard_id()]; + auto i = cache.find(host); + if (i != cache.end()) { + return i->second; + } + + auto j = _cfg->gcp_hosts().find(host); + if (j != _cfg->gcp_hosts().end()) { + auto result = ::make_shared(*this, host, j->second); + cache.emplace(host, result); + return result; + } + + throw std::invalid_argument("No such host: "+ host); + } + + + const encryption_config& config() const override { + return *_cfg; + } + shared_ptr get_config_encryption_key() const override { + return _cfg_encryption_key; + } + future<> load_config_encryption_key(const sstring & name) { + return get_system_key(name)->get_key().then([this](auto k) { + _cfg_encryption_key = std::move(k); + }); + } + /** + * This looks like checking too late, but since these are only used by + * replicated provider, they will be checked very early anyway, unless + * running tool mode, in which case they don't exist. + */ + template + T& check_service_object(T* t) const { + if (t == nullptr) { + throw std::runtime_error(fmt::format("Service {} not registered", typeid(T).name())); + } + return *t; + } + distributed& get_query_processor() const override { + return check_service_object(_qp); + } + distributed& get_storage_service() const override { + return check_service_object(_ss); + } + distributed& get_database() const override { + return check_service_object(_db); + } + distributed& get_migration_manager() const override { + return check_service_object(_mm); + } + + future<> start() override { + if (_qp && _ss && _db && _mm) { + co_await replicated_key_provider_factory::on_started(get_database().local(), get_migration_manager().local()); + } + } + future<> stop() override { + return smp::invoke_on_all([this]() -> future<> { + for (auto&& [id, h] : _per_thread_kmip_host_cache[this_shard_id()]) { + co_await h->disconnect(); + } + _per_thread_provider_cache[this_shard_id()].clear(); + _per_thread_system_key_cache[this_shard_id()].clear(); + _per_thread_kmip_host_cache[this_shard_id()].clear(); + _per_thread_kms_host_cache[this_shard_id()].clear(); + _per_thread_gcp_host_cache[this_shard_id()].clear(); + _per_thread_global_user_extension[this_shard_id()] = {}; + }); + } + + void add_global_user_encryption(shared_ptr ext) { + _per_thread_global_user_extension[this_shard_id()] = std::move(ext); + } + + shared_ptr get_global_user_encryption() const { + return _per_thread_global_user_extension[this_shard_id()]; + } + bool allow_per_table_encryption() const { + return _allow_per_table_encryption; + } +}; + +class encryption_schema_extension; + +std::ostream& operator<<(std::ostream& os, const encryption_schema_extension& ext); + +} + +template <> struct fmt::formatter : fmt::ostream_formatter {}; + +namespace encryption { + +class encryption_schema_extension : public schema_extension { + key_info _info; + shared_ptr _provider; + std::map _options; + std::optional _key_block_size; + + friend std::ostream& operator<<(std::ostream&, const encryption_schema_extension&); +public: + encryption_schema_extension(key_info, shared_ptr, std::map); + + using extension_ptr = ::shared_ptr; + + static extension_ptr create(encryption_context_impl&, std::map); + static extension_ptr create(encryption_context_impl&, const bytes&); + + static extension_ptr parse(encryption_context_impl& ctxt, db::extensions::schema_ext_config cfg) { + struct { + encryption_context_impl& _ctxt; + + extension_ptr operator()(const sstring&) const { + throw std::invalid_argument("Malformed extension"); + } + extension_ptr operator()(const std::map& opts) const { + return create(_ctxt, opts); + } + extension_ptr operator()(const bytes& v) const { + return create(_ctxt, v); + } + } v{ctxt}; + + auto res = std::visit(v, cfg); + // Note: We always allow _disbling_ per-table encryption, i.e. if user encryption is active, we fall back to node-local + if (res && !ctxt.allow_per_table_encryption() && ctxt.get_global_user_encryption()) { + throw std::invalid_argument(fmt::format("Node global user encryption is active and per-table encryption attributes have been prohibited ({})", *res)); + } + return res; + } + + static options parse_options(const bytes& v) { + return ser::deserialize_from_buffer(v, std::type_identity(), 0); + } + + future<::shared_ptr> key_for_read(opt_bytes id) const { + return _provider->key(_info, std::move(id)).then([](std::tuple k_id) { + return std::get<0>(std::move(k_id)); + }); + } + future, opt_bytes>> key_for_write(opt_bytes id = {}) const { + return _provider->key(_info, std::move(id)); + } + + bytes serialize() const override { + return ser::serialize_to_buffer(_options, 0); + } + future<> validate(const schema& s) const override { + try { + co_await _provider->validate(); + auto k = co_await key_for_write(); + logg.info("Added encryption extension to {}.{}", s.ks_name(), s.cf_name()); + logg.info(" Options: {}", _options); + logg.info(" Key Algorithm: {}", _info); + logg.info(" Provider: {}", *_provider); + + auto problems = std::get<0>(k)->validate_exact_info_result(); + if (!problems.empty()) { + logg.warn("{}", problems); + } + } catch (...) { + std::throw_with_nested(exceptions::configuration_exception((std::stringstream{} << "Validation failed:" << std::current_exception()).str())); + } + } + + bool should_delay_read(const opt_bytes& id) { + return _provider->should_delay_read(id); + } + size_t key_block_size() { + if (!_key_block_size) { + _key_block_size = symmetric_key(_info).block_size(); + } + return *_key_block_size; + } +}; + +std::ostream& operator<<(std::ostream& os, const encryption_schema_extension& ext) { + fmt::print(os, "{}, alg={}, provider={}", ext._options, ext._info, *ext._provider); + return os; +} + +encryption_schema_extension::encryption_schema_extension(key_info info, shared_ptr provider, std::map options) + : _info(std::move(info)) + , _provider(std::move(provider)) + , _options(std::move(options)) +{} + +::shared_ptr encryption_schema_extension::create(encryption_context_impl& ctxt, const bytes& v) { + auto map = parse_options(v); + return create(ctxt, map); +} + +::shared_ptr encryption_schema_extension::create(encryption_context_impl& ctxt, std::map map) { + key_info info = get_key_info(map); + auto provider = ctxt.get_provider(map); + if (!provider) { + return {}; + } + return ::make_shared(std::move(info), std::move(provider), std::move(map)); +} + +class encryption_file_io_extension : public sstables::file_io_extension { + ::shared_ptr _ctxt; +public: + encryption_file_io_extension(::shared_ptr ctxt) + : _ctxt(std::move(ctxt)) + {} + + attr_value_map get_attributes(const sstables::sstable& sst) const override { + auto& sc = sst.get_shared_components(); + if (!sc.scylla_metadata) { + return {}; + } + auto* exta = sc.scylla_metadata->get_extension_attributes(); + if (!exta) { + return {}; + } + + auto i = exta->map.find(encryption_attribute_ds); + if (i == exta->map.end()) { + return {}; + } + auto opts = encryption_schema_extension::parse_options(i->second.value); + + if (exta->map.count(key_id_attribute_ds)) { + auto id = exta->map.at(key_id_attribute_ds).value; + auto id_str = id.size() == utils::UUID::serialized_size() + ? sstring(fmt::format("{}", utils::UUID_gen::get_UUID(id))) + : to_hex(id) + ; + opts["key_id"] = std::move(id_str); + } + + if (exta->map.count(encrypted_components_attribute_ds)) { + std::vector ccs; + ccs.reserve(9); + auto mask = ser::deserialize_from_buffer(exta->map.at(encrypted_components_attribute_ds).value, std::type_identity{}, 0); + for (auto c : { sstables::component_type::Index, + sstables::component_type::CompressionInfo, + sstables::component_type::Data, + sstables::component_type::Summary, + sstables::component_type::Digest, + sstables::component_type::CRC, + sstables::component_type::Filter, + sstables::component_type::Statistics, + sstables::component_type::TemporaryStatistics, + }) { + if (mask & int(c)) { + ccs.emplace_back(c); + } + } + opts["components"] = fmt::to_string(fmt::join(ccs, ", ")); + } else { + opts["components"] = "Data"; + } + attr_value_map res; + res["encryption_info"] = std::move(opts); + return res; + } + + future wrap_file(sstables::sstable& sst, sstables::component_type type, file f, open_flags flags) override { + switch (type) { + case sstables::component_type::Scylla: + case sstables::component_type::TemporaryTOC: + case sstables::component_type::TOC: + co_return file{}; + default: + break; + } + + if (flags == open_flags::ro) { + // open existing. check read opts. + auto& sc = sst.get_shared_components(); + if (sc.scylla_metadata) { + auto* exta = sc.scylla_metadata->get_extension_attributes(); + if (exta) { + auto i = exta->map.find(encryption_attribute_ds); + // note: earlier builds of encryption extension would only encrypt data component, + // so iff we are opening old sstables we need to check if this component is actually + // encrypted. We use a bitmask attribute for this. + + bool ok = i != exta->map.end(); + if (ok && type != sstables::component_type::Data) { + ok = exta->map.count(encrypted_components_attribute_ds) && + (ser::deserialize_from_buffer(exta->map.at(encrypted_components_attribute_ds).value, std::type_identity{}, 0) & (1 << int(type))); + } + + if (ok) { + auto esx = encryption_schema_extension::create(*_ctxt, i->second.value); + opt_bytes id; + + if (exta->map.count(key_id_attribute_ds)) { + id = exta->map.at(key_id_attribute_ds).value; + } + + if (esx->should_delay_read(id)) { + logg.debug("Encrypted sstable component {} using delayed opening {} (id: {})", sst.component_basename(type), *esx, id); + + co_return make_delayed_encrypted_file(f, esx->key_block_size(), [esx, comp = sst.component_basename(type), id = std::move(id)] { + logg.trace("Delayed component {} using {} (id: {}) resolve", comp, *esx, id); + return esx->key_for_read(id); + }); + } + + logg.debug("Open encrypted sstable component {} using {} (id: {})", sst.component_basename(type), *esx, id); + + auto k = co_await esx->key_for_read(std::move(id)); + co_return make_encrypted_file(f, std::move(k)); + } + } + } + } else { + auto s = sst.get_schema(); + shared_ptr esx; + auto e = s->extensions().find(encryption_attribute); + // #4844 - don't allow schema encryption to be used for writing + // iff it is disallowed by config -> placeholder here + // (See schema_tables.cc::prepare_builder_from_table_row - if an extension + // is unavailable/non-creatable at load time a dummy object is inserted + // ) + if (e != s->extensions().end() && !e->second->is_placeholder()) { + esx = static_pointer_cast(e->second); + } else if (!is_system_keyspace(s->ks_name())) { + esx = _ctxt->get_global_user_encryption(); + } + if (esx) { + auto& sc = sst.get_shared_components(); + if (!sc.scylla_metadata) { + sc.scylla_metadata.emplace(); + } + auto& ext = sc.scylla_metadata->get_or_create_extension_attributes(); + opt_bytes id; + + // We are writing more than one component. If we used a named key before + // we need to make sure we use the exact same one for all components, + // even if something like KMIP key invalidation replaced it. + // This will also speed up key lookup in some cases, as both repl + // and kmip cache id bound keys. + if (ext.map.count(key_id_attribute_ds)) { + id = ext.map.at(key_id_attribute_ds).value; + } + + logg.debug("Write encrypted sstable component {} using {} (id: {})", sst.component_basename(type), *esx, id); + + /** + * #3954 We can be (and are) called with two components simultaneously (hello index, data). + * If this case we could block on the below "key" call and iff provider has certain cache behaviour (hello replicated) + * or caches expire, we could end up with different keys for respective components, leading to one + * of the components ending up unreadable. + */ + for (;;) { + auto [k, k_id] = co_await esx->key_for_write(std::move(id)); + + if (k_id && ext.map.count(key_id_attribute_ds)) { + id = ext.map.at(key_id_attribute_ds).value; + if (k_id != id) { + continue; + } + } + + id = std::move(k_id); + + if (!ext.map.count(encryption_attribute_ds)) { + ext.map.emplace(encryption_attribute_ds, sstables::disk_string{esx->serialize()}); + } + if (id) { + ext.map.emplace(key_id_attribute_ds, sstables::disk_string{*id}); + } + if (type != sstables::component_type::Data) { + uint32_t mask = 0; + if (ext.map.count(encrypted_components_attribute_ds)) { + mask = ser::deserialize_from_buffer(ext.map.at(encrypted_components_attribute_ds).value, std::type_identity{}, 0); + } + mask |= (1 << int(type)); + // just a marker. see above + ext.map[encrypted_components_attribute_ds] = sstables::disk_string{ser::serialize_to_buffer(mask, 0)}; + } + co_return make_encrypted_file(f, std::move(k)); + } + } + } + + co_return file{}; + } +}; + +std::string encryption_provider(const sstables::sstable& sst) { + auto& sc = sst.get_shared_components(); + if (!sc.scylla_metadata) { + return {}; + } + auto* exta = sc.scylla_metadata->get_extension_attributes(); + if (!exta) { + return {}; + } + + auto i = exta->map.find(encryption_attribute_ds); + if (i == exta->map.end()) { + return {}; + } + auto options = encryption_schema_extension::parse_options(i->second.value); + opt_wrapper opts(options); + + return opts(KEY_PROVIDER).value_or(std::string{}); +} + +namespace bfs = std::filesystem; + +class encryption_commitlog_file_extension : public db::commitlog_file_extension { + const ::shared_ptr _ctxt; + const options _opts; + + static const inline std::regex prop_expr = std::regex("^([^=]+)=(\\S+)$"); + static const inline sstring id_key = "key_id"; + static const inline std::string end_of_file_mark = "#-- end of file"; + +public: + encryption_commitlog_file_extension(::shared_ptr ctxt, options opts) + : _ctxt(ctxt) + , _opts(std::move(opts)) + {} + sstring config_name(const sstring& filename) const { + bfs::path p(filename); + auto dir = p.parent_path(); + auto file = p.filename(); + return (dir / bfs::path("." + file.string())).string(); + } + future wrap_file(const sstring& filename, file f, open_flags flags) override { + auto cfg_file = config_name(filename); + + if (flags == open_flags::ro) { + return file_exists(cfg_file).then([=, this](bool exists) { + if (!exists) { + // #1681 if file system errors caused the options file to simply not exist, + // we can at least hope that the file itself is not very encrypted either. + // But who knows. Will probably cause data corruption. + logg.info("Commitlog segment {} has no encryption info. Opening unencrypted.", filename); + return make_ready_future(std::move(f)); + } + return read_text_file_fully(cfg_file).then([f, this, filename](temporary_buffer buf) { + std::istringstream ss(std::string(buf.begin(), buf.end())); + options opts; + std::string line; + bool has_eof = false; + while (std::getline(ss, line)) { + std::smatch m; + if (std::regex_match(line, m, prop_expr)) { + auto k = m[1].str(); + auto v = m[2].str(); + opts[k] = v; + } else if (line == end_of_file_mark) { + has_eof = true; + } + } + + // #1682 - if we crashed while writing the options file, + // it is quite possible that we are eventually trying to + // open + replay an (empty) CL file, but cannot read the + // properties now, since _our_ metadata is empty/truncated + if (!has_eof) { + // just return the unwrapped file. + logg.info("Commitlog segment {} has incomplete encryption info. Opening unencrypted.", filename); + return make_ready_future(std::move(f)); + } + opt_bytes id; + if (opts.count(id_key)) { + id = base64_decode(opts[id_key]); + } + + auto provider = _ctxt->get_provider(opts); + + logg.debug("Open commitlog segment {} using {} (id: {})", filename, *provider, id); + auto info = make_shared(get_key_info(opts)); + return provider->key(*info, id).then([f, info](std::tuple, opt_bytes> k) { + return make_ready_future(make_encrypted_file(f, std::get<0>(k))); + }); + }); + }); + } else { + auto provider = _ctxt->get_provider(_opts); + auto info = make_shared(get_key_info(_opts)); + return provider->key(*info).then([f, this, info, cfg_file, filename, &provider = *provider](std::tuple, opt_bytes> k_id) { + auto&& k = std::get<0>(k_id); + auto&& id = std::get<1>(k_id); + std::ostringstream ss; + for (auto&p : _opts) { + ss << p.first << "=" << p.second << std::endl; + } + if (id) { + ss << id_key << "=" << base64_encode(*id) << std::endl; + } + ss << end_of_file_mark << std::endl; + + logg.debug("Creating commitlog segment {} using {} (id: {})", filename, provider, id); + + return write_text_file_fully(cfg_file, ss.str()).then([f, k] { + return make_ready_future(make_encrypted_file(f, k)); + }); + }); + } + } + future<> before_delete(const sstring& filename) override { + auto cfg_file = config_name(filename); + return file_exists(cfg_file).then([cfg_file](bool b) { + return b ? remove_file(cfg_file) : make_ready_future(); + }); + } +}; + +future> register_extensions(const db::config&, std::unique_ptr cfg_in, db::extensions& exts, const ::service_set& services) { + auto& cfg = *cfg_in; + auto ctxt = ::make_shared(std::move(cfg_in), services); + // Note: extensions are immutable and shared across shards. + // Object in them must be stateless. We anchor the context in the + // extension objects, and while it is not as such 100% stateless, + // it is close enough. + exts.add_schema_extension(encryption_attribute, [ctxt](auto v) { + return encryption_schema_extension::parse(*ctxt, std::move(v)); + }); + exts.add_sstable_file_io_extension(encryption_attribute, std::make_unique(ctxt)); + + auto maybe_get_options = [&](const utils::config_file::string_map& map, const sstring& what) -> std::optional { + options opts(map.begin(), map.end()); + opt_wrapper sie(opts); + if (!::strcasecmp(sie("enabled").value_or("false").c_str(), "false")) { + return std::nullopt; + } + // commitlog/system table encryption/global user encryption should not use replicated keys, + // We default to local keys, but KMIP/KMS is ok as well (better in fact). + opts[KEY_PROVIDER] = sie(KEY_PROVIDER).value_or(LOCAL_FILE_SYSTEM_KEY_PROVIDER_FACTORY); + if (opts[KEY_PROVIDER] == LOCAL_FILE_SYSTEM_KEY_PROVIDER_FACTORY && !sie(SECRET_KEY_FILE)) { + // system encryption uses different key folder than user tables. + // explicitly set the key file path + opts[SECRET_KEY_FILE] = (bfs::path(cfg.system_key_directory()) / bfs::path("system") / bfs::path(sie("key_name").value_or("system_table_keytab"))).string(); + } + // forbid replicated. we cannot guarantee being able to open sstables on populate + if (opts[KEY_PROVIDER] == REPLICATED_KEY_PROVIDER_FACTORY) { + throw std::invalid_argument("Replicated provider is not allowed for " + what); + } + return opts; + }; + + future<> f = make_ready_future<>(); + + auto opts = maybe_get_options(cfg.system_info_encryption(), "system table encryption"); + + if (opts) { + logg.info("Adding system info encryption using {}", *opts); + + exts.add_commitlog_file_extension(encryption_attribute, std::make_unique(ctxt, *opts)); + + // modify schemas for tables holding sensitive data to use encryption w. key described + // by the opts. + // since schemas are duplicated across shards, we must call to each shard and augument + // them all. + // Since we are in pre-init phase, this should be safe. + f = f.then([opts = *opts, &exts] { + return smp::invoke_on_all([opts = make_lw_shared(opts), &exts] () mutable { + auto& f = exts.schema_extensions().at(encryption_attribute); + for (auto& s : { db::system_keyspace::paxos(), db::system_keyspace::batchlog(), db::system_keyspace::dicts() }) { + exts.add_extension_to_schema(s, encryption_attribute, f(*opts)); + } + }); + }); + } + + if (cfg.config_encryption_active()) { + f = f.then([&cfg, ctxt] { + return ctxt->load_config_encryption_key(cfg.config_encryption_key_name()); + }); + } + + + if (!cfg.kmip_hosts().empty()) { + // only pre-create on shard 0. + f = f.then([&cfg, ctxt] { + return parallel_for_each(cfg.kmip_hosts(), [ctxt](auto& p) { + auto host = ctxt->get_kmip_host(p.first); + return host->connect(); + }); + }); + } + + if (!cfg.kms_hosts().empty()) { + // only pre-create on shard 0. + f = f.then([&cfg, ctxt] { + return parallel_for_each(cfg.kms_hosts(), [ctxt](auto& p) { + auto host = ctxt->get_kms_host(p.first); + return host->init(); + }); + }); + } + + if (!cfg.gcp_hosts().empty()) { + // only pre-create on shard 0. + f = f.then([&cfg, ctxt] { + return parallel_for_each(cfg.gcp_hosts(), [ctxt](auto& p) { + auto host = ctxt->get_gcp_host(p.first); + return host->init(); + }); + }); + } + + replicated_key_provider_factory::init(exts); + + auto user_opts = maybe_get_options(cfg.user_info_encryption(), "user table encryption"); + + if (user_opts) { + logg.info("Adding user info encryption using {}", *user_opts); + + f = f.then([user_opts = *user_opts, ctxt] { + return smp::invoke_on_all([user_opts = make_lw_shared(user_opts), ctxt]() { + auto ext = encryption_schema_extension::create(*ctxt, *user_opts); + ctxt->add_global_user_encryption(std::move(ext)); + }); + }); + } + + return f.then([ctxt]() -> ::shared_ptr { + return ctxt; + }); +} + +} + diff --git a/ent/encryption/encryption.hh b/ent/encryption/encryption.hh new file mode 100644 index 0000000000..d70fd3ae19 --- /dev/null +++ b/ent/encryption/encryption.hh @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include + +#include +#include +#include +#include + +#include +#include + +#include "../../bytes.hh" +#include "../../compress.hh" + +class service_set; + +namespace replica { +class database; +} + +namespace db { +class config; +class extensions; +} + +namespace cql3 { +class query_processor; +} +namespace service { +class storage_service; +class migration_manager; +} + +namespace sstables { + class sstable; +} + +namespace encryption { +inline const sstring KEY_PROVIDER = "key_provider"; +inline const sstring SECRET_KEY_PROVIDER_FACTORY_CLASS = "secret_key_provider_factory_class"; +inline const sstring SECRET_KEY_FILE = "secret_key_file"; +inline const sstring SYSTEM_KEY_FILE = "system_key_file"; +inline const sstring CIPHER_ALGORITHM = "cipher_algorithm"; +inline const sstring SECRET_KEY_STRENGTH = "secret_key_strength"; + +inline const sstring HOST_NAME = "kmip_host"; +inline const sstring TEMPLATE_NAME = "template_name"; +inline const sstring KEY_NAMESPACE = "key_namespace"; + +bytes base64_decode(const sstring&, size_t off = 0, size_t n = sstring::npos); +sstring base64_encode(const bytes&, size_t off = 0, size_t n = bytes::npos); +bytes calculate_md5(const bytes&, size_t off = 0, size_t n = bytes::npos); +bytes calculate_sha256(const bytes&, size_t off = 0, size_t n = bytes::npos); +bytes calculate_sha256(bytes_view); +bytes hmac_sha256(bytes_view msg, bytes_view key); + +future> read_text_file_fully(const sstring&); +future<> write_text_file_fully(const sstring&, temporary_buffer); +future<> write_text_file_fully(const sstring&, const sstring&); + +std::optional parse_expiry(std::optional); + +class symmetric_key; +struct key_info; + +using options = std::map; +using opt_bytes = std::optional; +using key_ptr = shared_ptr; + +/** + * wrapper for "options" (map) to provide an + * interface returning empty optionals for + * non-available values. Makes query simpler + * and allows .value_or(...)-statements, which + * are neat for default values. + * + * In the long run one could contemplate + * using non-std maps with similar built-in + * functionality for all our various configs + * in the system, but for now we are firmly + * entrenched in map + */ +template +class map_wrapper { + const Map& _options; +public: + using mapped_type = typename Map::mapped_type; + using key_type = typename Map::key_type; + + map_wrapper(const Map& opts) + : _options(opts) + {} + + std::optional operator()(const key_type& k) const { + auto i = _options.find(k); + if (i != _options.end()) { + return i->second; + } + return std::nullopt; + } +}; + +using opt_wrapper = map_wrapper; + +key_info get_key_info(const options&); + +class encryption_context; + +class key_provider { +public: + virtual ~key_provider() + {} + virtual future> key(const key_info&, opt_bytes = {}) = 0; + virtual future<> validate() const { + return make_ready_future<>(); + } + virtual bool should_delay_read(const opt_bytes&) const { + return false; + } +private: + friend std::ostream& operator<<(std::ostream&, const key_provider&); + virtual void print(std::ostream&) const = 0; +}; + +std::ostream& operator<<(std::ostream&, const key_provider&); + +} + +template <> struct fmt::formatter : fmt::ostream_formatter {}; + +namespace encryption { + +class key_provider_factory { +public: + virtual ~key_provider_factory() + {} + virtual shared_ptr get_provider(encryption_context& c, const options&) = 0; +}; + +class encryption_config; +class system_key; +class kmip_host; +class kms_host; +class gcp_host; + +/** + * Context is a singleton object, shared across shards. I.e. even though there are obvious mutating + * calls in it, it guarantees thread/shard safety. + * + * Why is this not a sharded thingamajing? Because its own instance methods need to send itself + * as a shard-safe object forwards, and thus need to know that same shard, which breaks the circle of + * ownership and stuff. + */ +class encryption_context { +public: + virtual ~encryption_context() = default; + virtual shared_ptr get_provider(const options&) = 0; + virtual shared_ptr get_system_key(const sstring&) = 0; + virtual shared_ptr get_kmip_host(const sstring&) = 0; + virtual shared_ptr get_kms_host(const sstring&) = 0; + virtual shared_ptr get_gcp_host(const sstring&) = 0; + + virtual shared_ptr get_cached_provider(const sstring& id) const = 0; + virtual void cache_provider(const sstring& id, shared_ptr) = 0; + + virtual const encryption_config& config() const = 0; + virtual shared_ptr get_config_encryption_key() const = 0; + + virtual distributed& get_query_processor() const = 0; + virtual distributed& get_storage_service() const = 0; + virtual distributed& get_database() const = 0; + virtual distributed& get_migration_manager() const = 0; + + sstring maybe_decrypt_config_value(const sstring&) const; + + virtual future<> start() = 0; + virtual future<> stop() = 0; +}; + +future> +register_extensions(const db::config&, std::unique_ptr, db::extensions&, const ::service_set&); + +// for testing +std::string encryption_provider(const sstables::sstable&); +} + diff --git a/ent/encryption/encryption_config.cc b/ent/encryption/encryption_config.cc new file mode 100644 index 0000000000..46e819025d --- /dev/null +++ b/ent/encryption/encryption_config.cc @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2015 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#include + +#include "db/config.hh" +#include "utils/config_file_impl.hh" + +#include "init.hh" +#include "encryption_config.hh" +#include "encryption.hh" + +#include + +encryption::encryption_config::encryption_config() + : config_file() +// BEGIN entry definitions + + , system_key_directory(this, "system_key_directory", value_status::Used, "/etc/scylla/conf/resources/system_keys", + R"foo(The directory where system keys are kept + +This directory should have 700 permissions and belong to the scylla user)foo") + + , config_encryption_active(this, "config_encryption_active", value_status::Used, false, "") + + , config_encryption_key_name(this, "config_encryption_key_name", value_status::Used, "system_key", + "Set to the local encryption key filename or KMIP key URL to use for configuration file property value decryption") + + , system_info_encryption(this, "system_info_encryption", value_status::Used, + { { "enabled", "false" }, { CIPHER_ALGORITHM, + "AES/CBC/PKCS5Padding" }, { + SECRET_KEY_STRENGTH, "128" }, + }, + R"foo(System information encryption settings + +If enabled, system tables that may contain sensitive information (system.batchlog, +system.paxos), hints files and commit logs are encrypted with the +encryption settings below. + +When enabling system table encryption on a node with existing data, run +`nodetool upgradesstables -a` on the listed tables to encrypt existing data. + +When tracing is enabled, sensitive info will be written into the tables in the +system_traces keyspace. Those tables should be configured to encrypt their data +on disk. + +It is recommended to use remote encryption keys from a KMIP server when using +Transparent Data Encryption (TDE) features. +Local key support is provided when a KMIP server is not available. + +See the scylla documentation for available key providers and their properties. +)foo") + , kmip_hosts(this, "kmip_hosts", value_status::Used, { }, + R"foo(KMIP host(s). + +The unique name of kmip host/cluster that can be referenced in table schema. + +host.yourdomain.com={ hosts=[, ...], keyfile=/path/to/keyfile, truststore=/path/to/truststore.pem, key_cache_millis=, timeout= }:... + +The KMIP connection management only supports failover, so all requests will go through a +single KMIP server. There is no load balancing, as no KMIP servers (at the time of this writing) +support read replication, or other strategies for availability. + +Hosts are tried in the order they appear here. Add them in the same sequence they'll fail over in. + +KMIP requests will fail over/retry 'max_command_retries' times (default 3) + +)foo") + , kms_hosts(this, "kms_hosts", value_status::Used, { }, + R"foo(KMS host(s). + +The unique name of kms host that can be referenced in table schema. + +host.yourdomain.com={ endpoint=, aws_access_key_id=, aws_secret_access_key=, aws_profile, aws_region=, aws_use_ec2_credentials, aws_use_ec2_region=, aws_assume_role_arn=, master_key=, keyfile=/path/to/keyfile, truststore=/path/to/truststore.pem, key_cache_millis=, timeout= }:... + +Actual connection can be either an explicit endpoint (:), or selected automatic via aws_region. + +If aws_use_ec2_region is true, regions is instead queried from EC2 metadata. + +Authentication can be explicit with aws_access_key_id and aws_secret_access_key. Either secret or both can be ommitted +in which case the provider will try to read them from AWS credentials in ~/.aws/credentials + +If aws_use_ec2_credentials is true, authentication is instead queried from EC2 metadata. + +If aws_assume_role_arn is set, scylla will issue an AssumeRole command and use the resulting security token for key operations. + +master_key is an AWS KMS key id or alias from which all keys used for actual encryption of scylla data will be derived. +This key must be pre-created with access policy allowing the above AWS id Encrypt, Decrypt and GenerateDataKey operations. + +)foo") + , gcp_hosts(this, "gcp_hosts", value_status::Used, { }, + R"foo(Google Compute Engine KMS host(s). + +The unique name of GCP kms host that can be referenced in table schema. + +gcp_project_id=, gcp_location=, master_key=, gcp_credentials_file=, gcp_impersonate_service_account=,keyfile=/path/to/keyfile, truststore=/path/to/truststore.pem, key_cache_millis=, timeout= }:... + +Authentication can be explicit with auth_file or by resolving default credentials (see google docs). + +If use_gcp_machine_credentials is true, authentication is instead queried from GCP metadata. + +auth_file can contain either a user, service or impersonated service account. + +master_key is an GCP KMS key name from which all keys used for actual encryption of scylla data will be derived. +This key must be pre-created with access policy allowing the above credentials Encrypt and Decrypt operations. + +)foo") + , user_info_encryption(this, "user_info_encryption", value_status::Used, + { { "enabled", "false" }, { CIPHER_ALGORITHM, + "AES/CBC/PKCS5Padding" }, { + SECRET_KEY_STRENGTH, "128" }, + }, + R"foo(Global user table encryption settings. If enabled, all user tables + will be encrypted using the provided settings, unless overridden + by table scylla_encryption_options.)foo") + , allow_per_table_encryption(this, "allow_per_table_encryption", value_status::Used, true, + "If 'user_info_encryption` is enabled this controls whether specifying per-table encryption using create/alter table is allowed" + ) + + +// END entry definitions +{} + +static class : public configurable { + std::unordered_map> _cfgs; + +public: + void append_options(db::config& cfg, boost::program_options::options_description_easy_init& init) override { + // While it is fine for normal execution to have just one, static (us) encryption config, + // it does not work well with unit testing, where we repeatedly create new cql_test_envs etc, + // since new config values will not be overwritten due to the actual named_values being shared here. + // Fix this (temporarily) by simply keeping a local map cfg->ecfg and using these. + // TODO: improve this by allowing db::config to hold named sub->configs (mapping config file objects). + if (_cfgs.count(&cfg)) { + throw std::runtime_error("Config already processed"); + } + auto& ccfg = _cfgs.emplace(&cfg, std::make_unique()).first->second; + // hook into main scylla.yaml. + cfg.add(ccfg->values()); + } + future initialize_ex(const boost::program_options::variables_map& opts, const db::config& cfg, db::extensions& exts, const service_set& services) override { + auto ccfg = _cfgs.count(&cfg) ? std::move(_cfgs.at(&cfg)) : std::make_unique(); + _cfgs.erase(&cfg); + auto ctxt = co_await encryption::register_extensions(cfg, std::move(ccfg), exts, services); + co_return [ctxt](system_state e) -> future<> { + switch (e) { + case system_state::started: + co_await ctxt->start(); + break; + case system_state::stopped: + co_await ctxt->stop(); + break; + default: + break; + } + }; + } +} cfg; diff --git a/ent/encryption/encryption_config.hh b/ent/encryption/encryption_config.hh new file mode 100644 index 0000000000..ced6f8ecf1 --- /dev/null +++ b/ent/encryption/encryption_config.hh @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include "../../utils/config_file.hh" + +namespace encryption { + +class encryption_config : public utils::config_file { +public: + encryption_config(); + + typedef std::unordered_map string_string_map; + + named_value system_key_directory; + named_value config_encryption_active; + named_value config_encryption_key_name; + named_value system_info_encryption; + named_value kmip_hosts; + named_value kms_hosts; + named_value gcp_hosts; + named_value user_info_encryption; + named_value allow_per_table_encryption; +}; + +} diff --git a/ent/encryption/encryption_exceptions.hh b/ent/encryption/encryption_exceptions.hh new file mode 100644 index 0000000000..7f551a7418 --- /dev/null +++ b/ent/encryption/encryption_exceptions.hh @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2024 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include "db/extensions.hh" + +namespace encryption { + +using base_error = db::extension_storage_exception; + +class permission_error : public db::extension_storage_permission_error { +public: + using mybase = db::extension_storage_permission_error; + using mybase::mybase; +}; + +class configuration_error : public db::extension_storage_misconfigured { +public: + using mybase = db::extension_storage_misconfigured; + using mybase::mybase; +}; + +class service_error : public base_error { +public: + using base_error::base_error; +}; + +class missing_resource_error : public db::extension_storage_resource_unavailable { +public: + using mybase = db::extension_storage_resource_unavailable; + using mybase::mybase; +}; + +// #4970 - not 100% correct, but network errors are +// generally intermittent/recoverable. Mark as a non-isolating +// error. +class network_error : public missing_resource_error { +public: + using missing_resource_error::missing_resource_error; +}; + +class malformed_response_error : public service_error { +public: + using service_error::service_error; +}; + +} + diff --git a/ent/encryption/gcp_host.cc b/ent/encryption/gcp_host.cc new file mode 100644 index 0000000000..23b2c39644 --- /dev/null +++ b/ent/encryption/gcp_host.cc @@ -0,0 +1,1031 @@ +/* + * Copyright (C) 2024 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#define CPP_JWT_USE_VENDORED_NLOHMANN_JSON +#include + +#include +#include +#include +#include "utils/to_string.hh" + +#include "gcp_host.hh" +#include "encryption.hh" +#include "encryption_exceptions.hh" +#include "symmetric_key.hh" +#include "utils/hash.hh" +#include "utils/loading_cache.hh" +#include "utils/UUID.hh" +#include "utils/UUID_gen.hh" +#include "utils/rjson.hh" +#include "marshal_exception.hh" +#include "db/config.hh" + +using namespace std::chrono_literals; +using namespace std::string_literals; + +logger gcp_log("gcp"); + +namespace encryption { +bool operator==(const gcp_host::credentials_source& k1, const gcp_host::credentials_source& k2) { + return k1.gcp_credentials_file == k2.gcp_credentials_file && k1.gcp_impersonate_service_account == k2.gcp_impersonate_service_account; +} +} + +template<> +struct fmt::formatter { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + auto format(const encryption::gcp_host::credentials_source& d, fmt::format_context& ctxt) const { + return fmt::format_to(ctxt.out(), "{{ gcp_credentials_file = {}, gcp_impersonate_service_account = {} }}", d.gcp_credentials_file, d.gcp_impersonate_service_account); + } +}; + +template<> +struct std::hash { + size_t operator()(const encryption::gcp_host::credentials_source& a) const { + return utils::tuple_hash{}(std::tie(a.gcp_credentials_file, a.gcp_impersonate_service_account)); + } +}; + +class encryption::gcp_host::impl { +public: + // set a rather long expiry. normal KMS policies are 365-day rotation of keys. + // we can do with 10 minutes. CMH. maybe even longer. + // (see comments below on what keys are here) + static inline constexpr std::chrono::milliseconds default_expiry = 600s; + static inline constexpr std::chrono::milliseconds default_refresh = 1200s; + + impl(encryption_context& ctxt, const std::string& name, const host_options& options) + : _ctxt(ctxt) + , _name(name) + , _options(options) + , _attr_cache(utils::loading_cache_config{ + .max_size = std::numeric_limits::max(), + .expiry = options.key_cache_expiry.value_or(default_expiry), + .refresh = options.key_cache_refresh.value_or(default_refresh)}, gcp_log, std::bind_front(&impl::create_key, this)) + , _id_cache(utils::loading_cache_config{ + .max_size = std::numeric_limits::max(), + .expiry = options.key_cache_expiry.value_or(default_expiry), + .refresh = options.key_cache_refresh.value_or(default_refresh)}, gcp_log, std::bind_front(&impl::find_key, this)) + {} + ~impl() = default; + + future<> init(); + const host_options& options() const { + return _options; + } + + future, id_type>> get_or_create_key(const key_info&, const option_override* = nullptr); + future> get_key_by_id(const id_type&, const key_info&, const option_override* = nullptr); + + using scopes_type = std::string; // space separated. avoids some transforms. makes other easy. +private: + class httpclient; + using key_and_id_type = std::tuple, id_type>; + + struct attr_cache_key { + credentials_source src; + std::string master_key; + key_info info; + bool operator==(const attr_cache_key& v) const = default; + }; + + friend struct fmt::formatter; + + struct attr_cache_key_hash { + size_t operator()(const attr_cache_key& k) const { + return utils::tuple_hash()(std::tie(k.master_key, k.src, k.info.len)); + } + }; + + struct id_cache_key { + credentials_source src; + id_type id; + bool operator==(const id_cache_key& v) const = default; + }; + + friend struct fmt::formatter; + + struct id_cache_key_hash { + size_t operator()(const id_cache_key& k) const { + return utils::tuple_hash()(std::tie(k.id, k.src)); + } + }; + + future create_key(const attr_cache_key&); + future find_key(const id_cache_key&); + + using timeout_clock = std::chrono::system_clock; + using timestamp_type = typename timeout_clock::time_point; + + struct access_token; + struct user_credentials; + struct service_account_credentials; + struct impersonated_service_account_credentials; + struct compute_engine_credentials{}; + + struct google_credentials; + + + struct access_token { + access_token() = default; + access_token(const rjson::value&); + + std::string token; + timestamp_type expiry; + scopes_type scopes; + + bool empty() const; + bool expired() const; + }; + + struct user_credentials { + user_credentials(const rjson::value&); + + std::string client_id; + std::string client_secret; + std::string refresh_token; + std::string access_token; + std::string quota_project_id; + }; + + using p_key = std::unique_ptr; + + struct service_account_credentials { + service_account_credentials(const rjson::value&); + + std::string client_id; + std::string client_email; + std::string private_key_id; + std::string private_key_pkcs8; + std::string token_server_uri; + std::string project_id; + std::string quota_project_id; + }; + + struct impersonated_service_account_credentials { + impersonated_service_account_credentials(std::string principal, google_credentials&&); + impersonated_service_account_credentials(const rjson::value&); + + std::vector delegates; + std::vector scopes; + std::string quota_project_id; + std::string iam_endpoint_override; + std::string target_principal; + + std::unique_ptr source_credentials; + access_token token; + }; + + using credentials_variant = std::variant< + user_credentials, + service_account_credentials, + impersonated_service_account_credentials, + compute_engine_credentials + >; + + struct google_credentials { + google_credentials(google_credentials&&) = default; + google_credentials(credentials_variant&& c) + : credentials(std::move(c)) + {} + google_credentials& operator=(google_credentials&&) = default; + credentials_variant credentials; + access_token token; + }; + + google_credentials from_data(std::string_view) const; + google_credentials from_data(const temporary_buffer& buf) const { + return from_data(std::string_view(buf.get(), buf.size())); + } + future from_file(const std::string& path) const { + auto buf = co_await read_text_file_fully(path); + co_return from_data(std::string_view(buf.get(), buf.size())); + } + + future get_default_credentials(); + + future get_access_token(const google_credentials&, const scopes_type& scopes) const; + + future<> refresh(google_credentials&, const scopes_type&) const; + + using key_values = std::initializer_list>; + + static std::string body(key_values kv); + + future send_request(std::string_view uri, std::string body, std::string_view content_type, httpd::operation_type = httpd::operation_type::GET, key_values headers = {}) const; + future send_request(std::string_view uri, const rjson::value& body, httpd::operation_type = httpd::operation_type::GET, key_values headers = {}) const; + future<> send_request(std::string_view uri, std::string body, std::string_view content_type, const std::function&, httpd::operation_type = httpd::operation_type::GET, key_values headers = {}) const; + + static std::tuple parse_key(std::string_view); + + future gcp_auth_post_with_retry(std::string_view uri, const rjson::value& body, const credentials_source&); + + encryption_context& _ctxt; + std::string _name; + host_options _options; + + std::unordered_map _cached_credentials; + + utils::loading_cache, attr_cache_key_hash> _attr_cache; + utils::loading_cache, id_cache_key_hash> _id_cache; + shared_ptr _creds; + std::unordered_map> _cache; + bool _initialized = false; + bool _checked_is_on_gce = false; + bool _is_on_gce = false; +}; + +template +static T get_option(const encryption::gcp_host::option_override* oov, std::optional C::* f, const T& def) { + if (oov) { + return (oov->*f).value_or(def); + } + return {}; +}; + +future, encryption::gcp_host::id_type>> encryption::gcp_host::impl::get_or_create_key(const key_info& info, const option_override* oov) { + attr_cache_key key { + .src = { + .gcp_credentials_file = get_option(oov, &option_override::gcp_credentials_file, _options.gcp_credentials_file), + .gcp_impersonate_service_account = get_option(oov, &option_override::gcp_impersonate_service_account, _options.gcp_impersonate_service_account), + }, + .master_key = get_option(oov, &option_override::master_key, _options.master_key), + .info = info, + }; + + if (key.master_key.empty()) { + throw configuration_error("No master key set in gcp host config or encryption attributes"); + } + try { + co_return co_await _attr_cache.get(key); + } catch (base_error&) { + throw; + } catch (std::invalid_argument& e) { + std::throw_with_nested(configuration_error(fmt::format("get_or_create_key: {}", e.what()))); + } catch (...) { + std::throw_with_nested(service_error(fmt::format("get_or_create_key: {}", std::current_exception()))); + } +} + +future> encryption::gcp_host::impl::get_key_by_id(const id_type& id, const key_info& info, const option_override* oov) { + // note: since KMS does not really have any actual "key" association of id -> key, + // we only cache/query raw bytes of some length. (See below). + // Thus keys returned are always new objects. But they are not huge... + id_cache_key key { + .src = { + .gcp_credentials_file = get_option(oov, &option_override::gcp_credentials_file, _options.gcp_credentials_file), + .gcp_impersonate_service_account = get_option(oov, &option_override::gcp_impersonate_service_account, _options.gcp_impersonate_service_account), + }, + .id = id, + }; + try { + auto data = co_await _id_cache.get(key); + co_return make_shared(info, data); + } catch (base_error&) { + throw; + } catch (std::invalid_argument& e) { + std::throw_with_nested(configuration_error(fmt::format("get_key_by_id: {}", e.what()))); + } catch (...) { + std::throw_with_nested(service_error(fmt::format("get_key_by_id: {}", std::current_exception()))); + } +} + +static const char CREDENTIAL_ENV_VAR[] = "GOOGLE_APPLICATION_CREDENTIALS"; +static const char WELL_KNOWN_CREDENTIALS_FILE[] = "application_default_credentials.json"; +static const char CLOUDSDK_CONFIG_DIRECTORY[] = "gcloud"; + +static const char USER_FILE_TYPE[] = "authorized_user"; +static const char SERVICE_ACCOUNT_FILE_TYPE[] = "service_account"; +static const char IMPERSONATED_SERVICE_ACCOUNT_FILE_TYPE[] = "impersonated_service_account"; + +static const char GCE_METADATA_HOST_ENV_VAR[] = "GCE_METADATA_HOST"; + +static const char DEFAULT_METADATA_SERVER_URL[] = "http://metadata.google.internal";; + +static const char METADATA_FLAVOR[] = "Metadata-Flavor"; +static const char GOOGLE[] = "Google"; + +static const char TOKEN_SERVER_URI[] = "https://oauth2.googleapis.com/token"; + +static const char AUTHORIZATION[] = "Authorization"; + +static const char KMS_SCOPE[] = "https://www.googleapis.com/auth/cloudkms"; +static const char CLOUD_PLATFORM_SCOPE[] = "https://www.googleapis.com/auth/cloud-platform"; + +//static const char[] CLOUD_SHELL_ENV_VAR = "DEVSHELL_CLIENT_PORT"; +//static const char[] SKIP_APP_ENGINE_ENV_VAR = "GOOGLE_APPLICATION_CREDENTIALS_SKIP_APP_ENGINE"; +//static const char[] NO_GCE_CHECK_ENV_VAR = "NO_GCE_CHECK"; +//static const char[] GCE_METADATA_HOST_ENV_VAR = "GCE_METADATA_HOST"; + +bool encryption::gcp_host::impl::access_token::empty() const { + return token.empty(); +} + +bool encryption::gcp_host::impl::access_token::expired() const { + if (empty()) { + return true; + } + return timeout_clock::now() >= this->expiry; +} + +encryption::gcp_host::impl::user_credentials::user_credentials(const rjson::value& v) + : client_id(rjson::get(v, "client_id")) + , client_secret(rjson::get(v, "client_secret")) + , refresh_token(rjson::get(v, "refresh_token")) + , quota_project_id(rjson::get_opt(v, "refresh_token").value_or("")) +{} + +encryption::gcp_host::impl::service_account_credentials::service_account_credentials(const rjson::value& v) + : client_id(rjson::get(v, "client_id")) + , client_email(rjson::get(v, "client_email")) + , private_key_id(rjson::get(v, "private_key_id")) + , private_key_pkcs8(rjson::get(v, "private_key")) + , token_server_uri([&] { + auto token_uri = rjson::get_opt(v, "token_uri"); + if (token_uri) { + // TODO: verify uri + return *token_uri; + } + return std::string{}; + }()) + , project_id(rjson::get_opt(v, "project_id").value_or("")) + , quota_project_id(rjson::get_opt(v, "refresh_token").value_or("")) +{} + + +encryption::gcp_host::impl::impersonated_service_account_credentials::impersonated_service_account_credentials(std::string principal, google_credentials&& c) + : target_principal(std::move(principal)) + , source_credentials(std::make_unique(std::move(c))) +{} + +encryption::gcp_host::impl::impersonated_service_account_credentials::impersonated_service_account_credentials(const rjson::value& v) + : delegates([&] { + std::vector res; + auto tmp = rjson::find(v, "delegates"); + if (tmp) { + if (!tmp->IsArray()) { + throw configuration_error("Malformed json"); + } + + for (const auto& d : tmp->GetArray()) { + res.emplace_back(std::string(rjson::to_string_view(d))); + } + } + return res; + }()) + , quota_project_id(rjson::get_opt(v, "quota_project_id").value_or("")) + , target_principal([&] { + auto url = rjson::get(v, "service_account_impersonation_url"); + + auto si = url.find_last_of('/'); + auto ei = url.find(":generateAccessToken"); + + if (si != std::string::npos && ei != std::string::npos && si < ei) { + return url.substr(si + 1, ei - si - 1); + } + throw configuration_error( "Unable to determine target principal from service account impersonation URL."); + }()) + , source_credentials([&]() -> decltype(source_credentials) { + auto& scjson = rjson::get(v, "source_credentials"); + auto type = rjson::get(scjson, "type"); + + if (type == USER_FILE_TYPE) { + return std::make_unique(user_credentials(scjson)); + } else if (type == SERVICE_ACCOUNT_FILE_TYPE) { + return std::make_unique(service_account_credentials(scjson)); + } + throw configuration_error(fmt::format("A credential of type {} is not supported as source credential for impersonation.", type)); + }()) +{} + +encryption::gcp_host::impl::google_credentials +encryption::gcp_host::impl::from_data(std::string_view content) const { + auto json = rjson::parse(content); + auto type = rjson::get_opt(json, "type"); + + if (!type) { + throw configuration_error("Error reading credentials from stream, 'type' field not specified."); + } + if (type == USER_FILE_TYPE) { + return google_credentials(user_credentials(json)); + } + if (type == SERVICE_ACCOUNT_FILE_TYPE) { + return google_credentials(service_account_credentials(json)); + } + if (type == IMPERSONATED_SERVICE_ACCOUNT_FILE_TYPE) { + return google_credentials(impersonated_service_account_credentials(json)); + } + throw configuration_error(fmt::format( + "Error reading credentials from stream, 'type' value '{}' not recognized. Expecting '{}', '{}' or '{}'." + , type, USER_FILE_TYPE, SERVICE_ACCOUNT_FILE_TYPE, IMPERSONATED_SERVICE_ACCOUNT_FILE_TYPE)); +} + +static std::string get_metadata_server_url() { + auto meta_host = std::getenv(GCE_METADATA_HOST_ENV_VAR); + auto token_uri = meta_host ? std::string("http://") + meta_host : DEFAULT_METADATA_SERVER_URL; + return token_uri; +} + +future +encryption::gcp_host::impl::get_default_credentials() { + auto credentials_path = std::getenv(CREDENTIAL_ENV_VAR); + + if (credentials_path != nullptr && strlen(credentials_path)) { + gcp_log.debug("Attempting to load credentials from file: {}", credentials_path); + + try { + co_return co_await from_file(credentials_path); + } catch (...) { + std::throw_with_nested(configuration_error(fmt::format( + "Error reading credential file from environment variable {}, value '{}'" + , CREDENTIAL_ENV_VAR + , credentials_path + )) + ); + } + } + + { + std::string well_known_file; + auto env_path = std::getenv("CLOUDSDK_CONFIG"); + if (env_path) { + well_known_file = fmt::format("~/{}/{}", env_path, WELL_KNOWN_CREDENTIALS_FILE); + } else { + well_known_file = fmt::format("~/.config/{}/{}", CLOUDSDK_CONFIG_DIRECTORY, WELL_KNOWN_CREDENTIALS_FILE); + } + + if (co_await seastar::file_exists(well_known_file)) { + gcp_log.debug("Attempting to load credentials from well known file: {}", well_known_file); + try { + co_return co_await from_file(well_known_file); + } catch (...) { + std::throw_with_nested(configuration_error(fmt::format( + "Error reading credential file from location {}" + , well_known_file + )) + ); + } + } + } + + { + // Then try Compute Engine and GAE 8 standard environment + gcp_log.debug("Attempting to load credentials from GCE"); + + auto is_on_gce = [this]() -> future { + if (_checked_is_on_gce) { + co_return _is_on_gce; + } + + auto token_uri = get_metadata_server_url(); + + for (int i = 1; i <= 3; ++i) { + try { + co_await send_request(token_uri, std::string{}, "", [&](const http::reply& rep, std::string_view) { + _checked_is_on_gce = true; + _is_on_gce = rep.get_header(METADATA_FLAVOR) == GOOGLE; + }, httpd::operation_type::GET, { { METADATA_FLAVOR, GOOGLE } }); + if (_checked_is_on_gce) { + co_return _is_on_gce;; + } + } catch (...) { + // TODO: handle timeout + break; + } + } + + auto linux_path = "/sys/class/dmi/id/product_name"; + if (co_await seastar::file_exists(linux_path)) { + auto f = file_desc::open(linux_path, O_RDONLY | O_CLOEXEC); + char buf[128] = {}; + f.read(buf, 128); + _is_on_gce = std::string_view(buf).find(GOOGLE) == 0; + } + + _checked_is_on_gce = true; + co_return _is_on_gce; + }; + + if (co_await is_on_gce()) { + co_return compute_engine_credentials{}; + } + } + + throw configuration_error("Could not determine initial credentials"); +} + +template +static void for_each_scope(const encryption::gcp_host::impl::scopes_type& s, Func&& f) { + size_t i = 0; + while(i < s.size()) { + auto j = s.find(' ', i + 1); + f(s.substr(i, j - i)); + i = j; + } +} + +encryption::gcp_host::impl::access_token::access_token(const rjson::value& json) + : token(rjson::get(json, "access_token")) + , expiry(timeout_clock::now() + std::chrono::seconds(rjson::get(json, "expires_in"))) + , scopes(rjson::get_opt(json, "scope").value_or("")) +{} + +std::string encryption::gcp_host::impl::body(key_values kv) { + std::ostringstream ss; + std::string_view sep = ""; + for (auto& [k, v] : kv) { + ss << sep << k << "=" << http::internal::url_encode(v); + sep = "&"; + } + return ss.str(); +} + +future encryption::gcp_host::impl::send_request(std::string_view uri, const rjson::value& body, httpd::operation_type op, key_values headers) const { + return send_request(uri, rjson::print(body), "application/json", op, std::move(headers)); +} + +future encryption::gcp_host::impl::send_request(std::string_view uri, std::string body, std::string_view content_type, httpd::operation_type op, key_values headers) const { + rjson::value v; + co_await send_request(uri, std::move(body), content_type, [&](const http::reply& rep, std::string_view s) { + if (rep._status != http::reply::status_type::ok) { + gcp_log.trace("Got unexpected reponse ({})", rep._status); + for (auto& [k, v] : rep._headers) { + gcp_log.trace("{}: {}", k, v); + } + gcp_log.trace("{}", s); + throw httpd::unexpected_status_error(rep._status); + } + v = rjson::parse(s); + }, op, std::move(headers)); + co_return v; +} + +future<> encryption::gcp_host::impl::send_request(std::string_view uri, std::string body, std::string_view content_type, const std::function& handler, httpd::operation_type op, key_values headers) const { + // Extremely simplified URI parsing. Does not handle any params etc. But we do not expect such here. + static boost::regex simple_url(R"foo((https?):\/\/([^\/:]+)(:\d+)?(\/.*)?)foo"); + + boost::smatch m; + std::string tmp(uri); + if (!boost::regex_match(tmp, m, simple_url)) { + throw std::invalid_argument(fmt::format("Could not parse URI {}", uri)); + } + + auto scheme = m[1].str(); + auto host = m[2].str(); + auto port = m[3].str(); + auto path = m[4].str(); + + auto addr = co_await net::dns::resolve_name(host, net::inet_address::family::INET /* CMH our client does not handle ipv6 well?*/); + auto certs = scheme == "https" + ? ::make_shared() + : shared_ptr() + ; + if (certs) { + if (!_options.priority_string.empty()) { + certs->set_priority_string(_options.priority_string); + } else { + certs->set_priority_string(db::config::default_tls_priority); + } + if (!_options.certfile.empty()) { + co_await certs->set_x509_key_file(_options.certfile, _options.keyfile, seastar::tls::x509_crt_format::PEM); + } + if (!_options.truststore.empty()) { + co_await certs->set_x509_trust_file(_options.truststore, seastar::tls::x509_crt_format::PEM); + } else { + co_await certs->set_system_trust(); + } + } + + uint16_t pi = port.empty() ? (certs ? 443 : 80) : uint16_t(std::stoi(port.substr(1))); + auto client = certs + ? http::experimental::client(socket_address(addr, pi), std::move(certs), host) + : http::experimental::client(socket_address(addr, pi)) + ; + if (path.empty()) { + path = "/"; + } + + gcp_log.trace("Resolved {} -> {}:{}{}", uri, addr, pi, path); + + auto req = http::request::make(op, host, path); + + for (auto& [k, v] : headers) { + req._headers[sstring(k)] = sstring(v); + } + + if (!body.empty()) { + if (content_type.empty()) { + content_type = "application/x-www-form-urlencoded"; + } + req.write_body("", std::move(body)); + req.set_mime_type(sstring(content_type)); + } + + gcp_log.trace("Sending {} request to {} ({}): {}", content_type, uri, headers, body); + + co_await client.make_request(std::move(req), [&] (const http::reply& rep, input_stream&& in) -> future<> { + auto&lh = handler; + auto lin = std::move(in); + auto result = co_await util::read_entire_stream_contiguous(lin); + gcp_log.trace("Got reponse {}: {}", int(rep._status), result); + lh(rep, result); + }); + + co_await client.close(); +} + + +future<> encryption::gcp_host::impl::refresh(google_credentials& c, const scopes_type& scopes) const { + if (!c.token.expired() && c.token.scopes == scopes) { + co_return; + } + c.token = co_await get_access_token(c, scopes); +} + +future +encryption::gcp_host::impl::get_access_token(const google_credentials& creds, const scopes_type& scope) const { + co_return co_await std::visit(overloaded_functor { + [&](const user_credentials& c) -> future { + assert(!c.refresh_token.empty()); + auto json = co_await send_request(TOKEN_SERVER_URI, body({ + { "client_id", c.client_id }, + { "client_secret", c.client_secret }, + { "refresh_token", c.refresh_token }, + { "grant_type", "grant_type" }, + }), "", httpd::operation_type::POST); + + co_return access_token{ json }; + }, + [&](const service_account_credentials& c) -> future { + using namespace jwt::params; + + jwt::jwt_object obj{algorithm("RS256"), secret(c.private_key_pkcs8), headers({{"kid", c.private_key_id }})}; + + auto uri = c.token_server_uri.empty() ? TOKEN_SERVER_URI : c.token_server_uri; + obj.add_claim("iss", c.client_email) + .add_claim("iat", timeout_clock::now()) + .add_claim("exp", timeout_clock::now() + std::chrono::seconds(3600)) + .add_claim("scope", scope) + .add_claim("aud", uri) + ; + auto sign = obj.signature(); + + auto json = co_await send_request(uri, body({ + { "grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer" }, + { "assertion", sign } + }), "", httpd::operation_type::POST); + co_return access_token{ json }; + }, + [&](const impersonated_service_account_credentials& c) -> future { + auto json_body = rjson::empty_object(); + auto scopes = rjson::empty_array(); + for_each_scope(scope, [&](std::string s) { + rjson::push_back(scopes, rjson::from_string(s)); + }); + + rjson::add(json_body, "scope", std::move(scopes)); + + if (!c.delegates.empty()) { + auto delegates = rjson::empty_array(); + for (auto& d : c.delegates) { + rjson::push_back(delegates, rjson::from_string(d)); + } + rjson::add(json_body, "delegates", std::move(delegates)); + } + + rjson::add(json_body, "lifetime", "3600s"); + + co_await refresh(*c.source_credentials, CLOUD_PLATFORM_SCOPE); + + auto endpoint = c.iam_endpoint_override.empty() + ? fmt::format("https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:generateAccessToken", c.target_principal) + : c.iam_endpoint_override + ; + auto json = co_await send_request(endpoint, json_body, httpd::operation_type::POST, { + { AUTHORIZATION, fmt::format("Bearer {}", c.source_credentials->token.token) }, + }); + + struct tm tmp; + ::strptime(rjson::get(json, "expireTime").data(), "%FT%TZ", &tmp); + + access_token a; + + a.expiry = timeout_clock::from_time_t(::mktime(&tmp)); + a.scopes = scope; + a.token = rjson::get(json, "accessToken"); + + co_return a; + }, + [this](const compute_engine_credentials& c) -> future { + auto meta_uri = get_metadata_server_url(); + auto token_uri = meta_uri + "/computeMetadata/v1/instance/service-accounts/default/token"; + try { + auto json = co_await send_request(token_uri, std::string{}, "", httpd::operation_type::GET, { { METADATA_FLAVOR, GOOGLE } }); + co_return access_token{ json }; + } catch (...) { + std::throw_with_nested(service_error("Unexpected Error code trying to get security access token from Compute Engine metadata for the default service account")); + } + } + }, creds.credentials); +} + +future encryption::gcp_host::impl::gcp_auth_post_with_retry(std::string_view uri, const rjson::value& body, const credentials_source& src) { + auto i = _cached_credentials.find(src); + if (i == _cached_credentials.end()) { + try { + auto c = !src.gcp_credentials_file.empty() + ? co_await from_file(src.gcp_credentials_file) + : co_await get_default_credentials() + ; + if (!src.gcp_credentials_file.empty()) { + gcp_log.trace("Loaded credentials from {}", src.gcp_credentials_file); + } + if (!src.gcp_impersonate_service_account.empty()) { + c = google_credentials(impersonated_service_account_credentials(src.gcp_impersonate_service_account, std::move(c))); + } + i = _cached_credentials.emplace(src, std::move(c)).first; + } catch (...) { + gcp_log.warn("Error resolving credentials for {}: {}", src, std::current_exception()); + throw; + } + } + + assert(i != _cached_credentials.end()); // should either be set now or we threw. + + auto& creds = i->second; + + int retries = 0; + + for (;;) { + try { + co_await this->refresh(creds, KMS_SCOPE); + } catch (...) { + std::throw_with_nested(permission_error("Error refreshing credentials")); + } + + try { + auto res = co_await send_request(uri, body, httpd::operation_type::POST, { + { AUTHORIZATION, fmt::format("Bearer {}", creds.token.token) }, + }); + co_return res; + } catch (httpd::unexpected_status_error& e) { + gcp_log.debug("{}: Got unexpected response: {}", uri, e.status()); + if (e.status() == http::reply::status_type::unauthorized && retries++ < 3) { + // refresh access token and retry. + continue; + } + if (e.status() == http::reply::status_type::unauthorized) { + std::throw_with_nested(permission_error(std::string(uri))); + } + std::throw_with_nested(service_error(std::string(uri))); + } catch (...) { + std::throw_with_nested(network_error(std::string(uri))); + } + } +} + +static constexpr char GCP_KMS_QUERY_TEMPLATE[] = "https://cloudkms.googleapis.com/v1/projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}:{}"; + +future<> encryption::gcp_host::impl::init() { + if (_initialized) { + co_return; + } + + if (!_options.master_key.empty()) { + gcp_log.debug("Looking up master key"); + + attr_cache_key k{ + .src = _options, + .master_key = _options.master_key, + .info = key_info{ .alg = "AES", .len = 128 }, + }; + co_await create_key(k); + gcp_log.debug("Master key exists"); + } else { + gcp_log.info("No default master key configured. Not verifying."); + } + _initialized = true; +} + +std::tuple encryption::gcp_host::impl::parse_key(std::string_view spec) { + auto i = spec.find_last_of('/'); + if (i == std::string_view::npos) { + throw std::invalid_argument(fmt::format("Invalid master key spec '{}'. Must be in format /", spec)); + } + return std::make_tuple(std::string(spec.substr(0, i)), std::string(spec.substr(i + 1))); +} + +future encryption::gcp_host::impl::create_key(const attr_cache_key& k) { + auto& info = k.info; + + /** + * Google GCP KMS does allow us to create keys, but like AWS this would + * force us to deal with permissions and assignments etc. We instead + * require a pre-prepared key. + * + * Like AWS, we cannot get the actual key out, nor can we really bulk + * encrypt/decrypt things. So we do just like with AWS KMS, and generate + * a data key, and encrypt it as the key ID. + * + * For ID -> key, we simply split the ID into the encrypted key part, and + * the master key name part, decrypt the first using the second (AWS KMS Decrypt), + * and create a local key using the result. + * + * Data recovery: + * Assuming you have data encrypted using a KMS generated key, you will have + * metadata detailing algorithm, key length etc (see sstable metadata, and key info). + * Metadata will also include a byte blob representing the ID of the encryption key. + * For GCP KMS, the ID will actually be a text string: + * :: + * + * I.e. something like: + * mykeyring:mykey:e56sadfafa3324ff=/wfsdfwssdf + * + * The actual data key can be retreived by doing a KMS "Decrypt" of the data blob part + * using the KMS key referenced by the key ID. This gives back actual key data that can + * be used to create a symmetric_key with algo, length etc as specified by metadata. + * + */ + + // avoid creating too many keys and too many calls. If we are not shard 0, delegate there. + if (this_shard_id() != 0) { + auto [data, id] = co_await smp::submit_to(0, [this, k]() -> future> { + auto host = _ctxt.get_gcp_host(_name); + auto [key, id] = co_await host->_impl->_attr_cache.get(k); + co_return std::make_tuple(key != nullptr ? key->key() : bytes{}, id); + }); + co_return key_and_id_type{ + data.empty() ? nullptr : make_shared(info, data), + id + }; + } + + // note: since external keys are _not_ stored, + // there is nothing we can "look up" or anything. Always + // new key here. + + gcp_log.debug("Creating new key: {}", info); + + auto [keyring, keyname] = parse_key(k.master_key); + + auto key = make_shared(info); + auto url = fmt::format(GCP_KMS_QUERY_TEMPLATE, + _options.gcp_project_id, + _options.gcp_location, + keyring, + keyname, + "encrypt" + ); + auto query = rjson::empty_object(); + rjson::add(query, "plaintext", std::string(base64_encode(key->key()))); + + auto response = co_await gcp_auth_post_with_retry(url, query, k.src); + auto cipher = rjson::get(response, "ciphertext"); + auto data = base64_decode(cipher); + + auto sid = fmt::format("{}/{}:{}", keyring, keyname, cipher); + bytes id(sid.begin(), sid.end()); + + gcp_log.trace("Created key id {}", sid); + + co_return key_and_id_type{ key, id }; +} + +future encryption::gcp_host::impl::find_key(const id_cache_key& k) { + // avoid creating too many keys and too many calls. If we are not shard 0, delegate there. + if (this_shard_id() != 0) { + co_return co_await smp::submit_to(0, [this, k]() -> future { + auto host = _ctxt.get_gcp_host(_name); + auto bytes = co_await host->_impl->_id_cache.get(k); + co_return bytes; + }); + } + + // See create_key. ID consists of :. + // master id can contain ':', but blob will not. + // (we are being wasteful, and keeping the base64 encoding - easier to read) + std::string_view id(reinterpret_cast(k.id.data()), k.id.size()); + gcp_log.debug("Finding key: {}", id); + + auto pos = id.find_last_of(':'); + auto pos2 = id.find_last_of('/', pos - 1); + if (pos == id_type::npos || pos2 == id_type::npos || pos2 >= pos) { + throw std::invalid_argument(fmt::format("Not a valid key id: {}", id)); + } + + std::string keyring(id.begin(), id.begin() + pos2); + std::string keyname(id.begin() + pos2 + 1, id.begin() + pos); + std::string enc(id.begin() + pos + 1, id.end()); + + auto url = fmt::format(GCP_KMS_QUERY_TEMPLATE, + _options.gcp_project_id, + _options.gcp_location, + keyring, + keyname, + "decrypt" + ); + auto query = rjson::empty_object(); + rjson::add(query, "ciphertext", enc); + + auto response = co_await gcp_auth_post_with_retry(url, query, k.src); + auto data = base64_decode(rjson::get(response, "plaintext")); + + // we know nothing about key type etc, so just return data. + co_return data; +} + +encryption::gcp_host::gcp_host(encryption_context& ctxt, const std::string& name, const host_options& options) + : _impl(std::make_unique(ctxt, name, options)) +{} + +encryption::gcp_host::gcp_host(encryption_context& ctxt, const std::string& name, const std::unordered_map& map) + : gcp_host(ctxt, name, [&map] { + host_options opts; + map_wrapper> m(map); + + opts.master_key = m("master_key").value_or(""); + + opts.gcp_project_id = m("gcp_project_id").value_or(""); + opts.gcp_location = m("gcp_location").value_or(""); + + opts.gcp_credentials_file = m("gcp_credentials_file").value_or(""); + opts.gcp_impersonate_service_account = m("gcp_impersonate_service_account").value_or(""); + + opts.certfile = m("certfile").value_or(""); + opts.keyfile = m("keyfile").value_or(""); + opts.truststore = m("truststore").value_or(""); + opts.priority_string = m("priority_string").value_or(""); + + opts.key_cache_expiry = parse_expiry(m("key_cache_expiry")); + opts.key_cache_refresh = parse_expiry(m("key_cache_refresh")); + + return opts; + }()) +{} + +encryption::gcp_host::~gcp_host() = default; + +future<> encryption::gcp_host::init() { + return _impl->init(); +} + +const encryption::gcp_host::host_options& encryption::gcp_host::options() const { + return _impl->options(); +} + +future, encryption::gcp_host::id_type>> encryption::gcp_host::get_or_create_key(const key_info& info, const option_override* oov) { + return _impl->get_or_create_key(info, oov); +} + +future> encryption::gcp_host::get_key_by_id(const id_type& id, const key_info& info, const option_override* oov) { + return _impl->get_key_by_id(id, info, oov); +} + +template<> +struct fmt::formatter { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + auto format(const encryption::gcp_host::impl::attr_cache_key& d, fmt::format_context& ctxt) const { + return fmt::format_to(ctxt.out(), "{},{},{}", d.master_key, d.src.gcp_credentials_file, d.src.gcp_impersonate_service_account); + } +}; + +template<> +struct fmt::formatter { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + auto format(const encryption::gcp_host::impl::id_cache_key& d, fmt::format_context& ctxt) const { + return fmt::format_to(ctxt.out(), "{},{},{}", d.id, d.src.gcp_credentials_file, d.src.gcp_impersonate_service_account); + } +}; diff --git a/ent/encryption/gcp_host.hh b/ent/encryption/gcp_host.hh new file mode 100644 index 0000000000..8764c0d24a --- /dev/null +++ b/ent/encryption/gcp_host.hh @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2024 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +#include "symmetric_key.hh" + +namespace encryption { + +class encryption_context; +struct key_info; + +class gcp_host { +public: + class impl; + + template + struct t_credentials_source { + // Path to credentials JSON file (exported from gcloud console) + T gcp_credentials_file; + // Optional service account (email address) to impersonate + T gcp_impersonate_service_account; + }; + + using credentials_source = t_credentials_source; + + struct host_options : public credentials_source { + std::string gcp_project_id; + std::string gcp_location; + + // GCP KMS Key to encrypt data keys with. Format: / + std::string master_key; + + // tls. if unspeced, use system for https + // GCP does not (afaik?) allow certificate auth + // but we keep the option available just in case. + std::string certfile; + std::string keyfile; + std::string truststore; + std::string priority_string; + + std::optional key_cache_expiry; + std::optional key_cache_refresh; + }; + + using id_type = bytes; + + gcp_host(encryption_context&, const std::string& name, const host_options&); + gcp_host(encryption_context&, const std::string& name, const std::unordered_map&); + ~gcp_host(); + + future<> init(); + const host_options& options() const; + + struct option_override : public t_credentials_source> { + std::optional master_key; + }; + + future, id_type>> get_or_create_key(const key_info&, const option_override* = nullptr); + future> get_key_by_id(const id_type&, const key_info&, const option_override* = nullptr); +private: + std::unique_ptr _impl; +}; + +} diff --git a/ent/encryption/gcp_key_provider.cc b/ent/encryption/gcp_key_provider.cc new file mode 100644 index 0000000000..1c00b969ac --- /dev/null +++ b/ent/encryption/gcp_key_provider.cc @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2024 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#include +#include + +#include "gcp_key_provider.hh" +#include "gcp_host.hh" + +namespace encryption { + +class gcp_key_provider : public key_provider { +public: + gcp_key_provider(::shared_ptr gcp_host, std::string name, gcp_host::option_override oov) + : _gcp_host(std::move(gcp_host)) + , _name(std::move(name)) + , _oov(std::move(oov)) + {} + future> key(const key_info& info, opt_bytes id) override { + if (id) { + return _gcp_host->get_key_by_id(*id, info, &_oov).then([id](key_ptr k) { + return make_ready_future>(std::tuple(k, id)); + }); + } + return _gcp_host->get_or_create_key(info, &_oov).then([](std::tuple k_id) { + return make_ready_future>(k_id); + }); + } + void print(std::ostream& os) const override { + os << _name; + } +private: + ::shared_ptr _gcp_host; + std::string _name; + gcp_host::option_override _oov; +}; + +shared_ptr gcp_key_provider_factory::get_provider(encryption_context& ctxt, const options& map) { + opt_wrapper opts(map); + auto gcp_host = opts("gcp_host"); + + + gcp_host::option_override oov { + .master_key = opts("master_key"), + }; + + oov.gcp_credentials_file = opts("gcp_credentials_file"); + oov.gcp_impersonate_service_account = opts("gcp_impersonate_service_account"); + + if (!gcp_host) { + throw std::invalid_argument("gcp_host must be provided"); + } + + auto host = ctxt.get_gcp_host(*gcp_host); + auto id = gcp_host.value() + + ":" + oov.master_key.value_or(host->options().master_key) + + ":" + oov.gcp_credentials_file.value_or(host->options().gcp_credentials_file) + + ":" + oov.gcp_impersonate_service_account.value_or(host->options().gcp_impersonate_service_account) + ; + + auto provider = ctxt.get_cached_provider(id); + + if (!provider) { + provider = ::make_shared(host, *gcp_host, std::move(oov)); + ctxt.cache_provider(id, provider); + } + + return provider; +} + +} diff --git a/ent/encryption/gcp_key_provider.hh b/ent/encryption/gcp_key_provider.hh new file mode 100644 index 0000000000..76ddb674f6 --- /dev/null +++ b/ent/encryption/gcp_key_provider.hh @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2024 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include "encryption.hh" +#include "system_key.hh" + +namespace encryption { + +class gcp_key_provider_factory : public key_provider_factory { +public: + shared_ptr get_provider(encryption_context&, const options&) override; +}; + +/** + * See comment for AWS KMS regarding system key support. + */ +} diff --git a/ent/encryption/kmip_host.cc b/ent/encryption/kmip_host.cc new file mode 100644 index 0000000000..1e2bf5dd0b --- /dev/null +++ b/ent/encryption/kmip_host.cc @@ -0,0 +1,1222 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#ifdef HAVE_KMIP + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +// workaround cryptsoft sdk issue: +#define strcasestr kmip_strcasestr +#include +#include +#undef strcasestr + +#include "kmip_host.hh" +#include "encryption.hh" +#include "encryption_exceptions.hh" +#include "symmetric_key.hh" +#include "utils/hash.hh" +#include "utils/loading_cache.hh" +#include "utils/UUID.hh" +#include "utils/UUID_gen.hh" +#include "marshal_exception.hh" +#include "db/config.hh" + +using namespace std::chrono_literals; + +static logger kmip_log("kmip"); +static constexpr uint16_t kmip_port = 5696u; +// default for command execution/failover retry. +static constexpr int default_num_cmd_retry = 5; +static constexpr int min_num_cmd_retry = 2; +static constexpr auto base_backoff_time = 100ms; + +std::ostream& operator<<(std::ostream& os, KMIP* kmip) { + auto* s = KMIP_dump_str(kmip, KMIP_DUMP_FORMAT_DEFAULT); + os << s; + free(s); + return os; +} + +static void kmip_logger(void *cb_arg, unsigned char *str, unsigned long len) { + // kmipc likes to write a log of white space and newlines. Skip these. + std::string_view v(reinterpret_cast(str), len); + if (std::find_if(v.begin(), v.end(), [](char c) { return !::isspace(c); }) == v.end()) { + return; + } + kmip_log.trace("kmipcmd: {}", v); +} + +namespace encryption { + +bool operator==(const kmip_host::key_options& l, const kmip_host::key_options& r) { + return std::tie(l.template_name, l.key_namespace) == std::tie(r.template_name, r.key_namespace); +} + +class kmip_error_category : public std::error_category { +public: + constexpr kmip_error_category() noexcept : std::error_category{} {} + const char * name() const noexcept { + return "KMIP"; + } + std::string message(int error) const { + return KMIP_error2string(error); + } +}; + +static const kmip_error_category kmip_errorc; + +class kmip_error : public std::system_error { +public: + kmip_error(int res) + : system_error(res, kmip_errorc) + {} + kmip_error(int res, const std::string& msg) + : system_error(res, kmip_errorc, msg) + {} +}; + +// Checks a gnutls return value. +// < 0 -> error. +static void kmip_chk(int res, KMIP_CMD * cmd = nullptr) { + if (res != KMIP_ERROR_NONE) { + int status=0, reason=0; + char* message = nullptr; + + if (KMIP_CMD_get_result(cmd, &status, &reason, &message) == KMIP_ERROR_NONE) { + auto* ctxt = cmd != nullptr ? KMIP_CMD_get_ctx(cmd) : "(unknown cmd)"; + auto s = fmt::format("{}: status={}, reason={}, message={}", + ctxt, + KMIP_RESULT_STATUS_to_string(status, 0, nullptr), + KMIP_RESULT_REASON_to_string(reason, 0, nullptr), + message ? message : "" + ); + throw kmip_error(res, s); + } + throw kmip_error(res); + } +} + + +class kmip_host::impl { +public: + struct kmip_key_info { + key_info info; + key_options options; + bool operator==(const kmip_key_info& i) const { + return info == i.info && options == i.options; + } + friend std::ostream& operator<<(std::ostream& os, const kmip_key_info& info) { + return os << info.info << ":" << info.options; + } + }; + struct kmip_key_info_hash { + size_t operator()(const kmip_key_info& i) const { + return utils::tuple_hash()( + std::tie(i.info.alg, i.info.len, + i.options.template_name, + i.options.key_namespace)); + } + }; + + using key_and_id_type = std::tuple, id_type>; + + inline static constexpr std::chrono::milliseconds default_expiry = 30s; + inline static constexpr std::chrono::milliseconds default_refresh = 100s; + inline static constexpr uintptr_t max_hosts = 1<<8; + + inline static constexpr size_t def_max_pooled_connections_per_host = 8; + + impl(encryption_context& ctxt, const sstring& name, const host_options& options) + : _ctxt(ctxt), _name(name), _options(options), _attr_cache( + utils::loading_cache_config{ + .max_size = std::numeric_limits::max(), + .expiry = options.key_cache_expiry.value_or( + default_expiry), + .refresh = options.key_cache_refresh.value_or(default_refresh)}, + kmip_log, + std::bind(&impl::create_key, this, + std::placeholders::_1)), + _id_cache( + utils::loading_cache_config{ + .max_size = std::numeric_limits::max(), + .expiry = options.key_cache_expiry.value_or( + default_expiry), + .refresh = options.key_cache_refresh.value_or(default_refresh), + }, + kmip_log, + std::bind(&impl::find_key, this, + std::placeholders::_1)), + _max_retry(std::max(size_t(min_num_cmd_retry), options.max_command_retries.value_or(default_num_cmd_retry))) + { + if (_options.hosts.size() > max_hosts) { + throw std::invalid_argument("Too many hosts"); + } + + KMIP_CMD_set_default_logfile(nullptr, nullptr); // disable logfile + KMIP_CMD_set_default_logger(kmip_logger, nullptr); // send logs to us instead + } + + future<> connect(); + future<> disconnect(); + future, id_type>> get_or_create_key(const key_info&, const key_options& = {}); + future> get_key_by_id(const id_type&, const std::optional& = {}); + + id_type kmip_id_to_id(const sstring&) const; + sstring id_to_kmip_string(const id_type&) const; +private: + future create_key(const kmip_key_info&); + future> find_key(const id_type&); + future> find_matching_keys(const kmip_key_info&, std::optional max = {}); + + static shared_ptr ensure_compatible_key(shared_ptr, const key_info&); + + template + class kmip_handle; + class kmip_cmd; + class kmip_data_list; + class connection; + + std::tuple make_attributes(const kmip_key_info&, bool include_template = true) const; + + union userdata { + void * ptr; + const char* host; + }; + + friend std::ostream& operator<<(std::ostream& os, const impl& me) { + fmt::print(os, "{}", me._name); + return os; + } + + using con_ptr = ::shared_ptr; + using opt_int = std::optional; + + template + future do_cmd(kmip_cmd, Func &&); + template + future do_cmd(KMIP_CMD*, con_ptr, Func&, bool retain_connection_after_command = false); + + future get_connection(KMIP_CMD*); + future get_connection(const sstring&); + future<> clear_connections(const sstring& host); + + void release(KMIP_CMD*, con_ptr, bool retain_connection = false); + + size_t max_pooled_connections_per_host() const { + return _options.max_pooled_connections_per_host.value_or(def_max_pooled_connections_per_host); + } + bool is_current_host(const sstring& host) { + return host == _options.hosts.at(_index % _options.hosts.size()); + } + + encryption_context& _ctxt; + sstring _name; + host_options _options; + utils::loading_cache, + kmip_key_info_hash> _attr_cache; + + utils::loading_cache, 2, + utils::loading_cache_reload_enabled::yes, + utils::simple_entry_size<::shared_ptr>> _id_cache; + + using connections = std::deque; + using host_to_connections = std::unordered_map; + + host_to_connections _host_connections; + // current default host. If a host fails, incremented and + // we try another in the host ip list. + size_t _index = 0; + size_t _max_retry = default_num_cmd_retry; +}; + +} + +template <> struct fmt::formatter : fmt::ostream_formatter {}; +template <> struct fmt::formatter : fmt::ostream_formatter {}; + +namespace encryption { + +class kmip_host::impl::connection { +public: + connection(const sstring& host, host_options& options) + : _host(host) + , _options(options) + {} + ~connection() + {} + + const sstring& host() const { + return _host; + } + + void attach(KMIP_CMD*); + + future<> connect(); + future<> wait_for_io(); + future<> close(); +private: + static int io_callback(KMIP*, void*, int, void*, unsigned int, unsigned int*); + + int send(void*, unsigned int, unsigned int*); + int recv(void*, unsigned int, unsigned int*); + + friend std::ostream& operator<<(std::ostream& os, const connection& me) { + return os << me._host; + } + + sstring _host; + host_options& _options; + output_stream _output; + input_stream _input; + seastar::connected_socket _socket; + std::optional> _in_buffer; + std::optional> _pending; +}; + +} + +template <> struct fmt::formatter : fmt::ostream_formatter {}; + +namespace encryption { + +future<> kmip_host::impl::connection::connect() { + auto cred = ::make_shared(); + auto f = make_ready_future(); + + kmip_log.debug("connecting {}", _host); + + if (!_options.priority_string.empty()) { + cred->set_priority_string(_options.priority_string); + } else { + cred->set_priority_string(db::config::default_tls_priority); + } + + if (!_options.certfile.empty()) { + f = f.then([this, cred] { + return cred->set_x509_key_file(_options.certfile, _options.keyfile, seastar::tls::x509_crt_format::PEM); + }); + } + if (!_options.truststore.empty()) { + f = f.then([this, cred] { + return cred->set_x509_trust_file(_options.truststore, seastar::tls::x509_crt_format::PEM); + }); + } + return f.then([this, cred] { + // TODO, find if we should do hostname verification + // TODO: connect all failovers already? + + auto i = _host.find_last_of(':'); + auto name = _host.substr(0, i); + auto port = i != sstring::npos ? std::stoul(_host.substr(i + 1)) : kmip_port; + + return seastar::net::dns::resolve_name(name).then([this, cred, port](seastar::net::inet_address addr) { + return seastar::tls::connect(cred, seastar::ipv4_addr{addr, uint16_t(port)}).then([this](seastar::connected_socket s) { + kmip_log.debug("Successfully connected {}", _host); + // #998 Set keepalive to try avoiding connection going stale inbetween commands. + s.set_keepalive_parameters(net::tcp_keepalive_params{60s, 60s, 10}); + s.set_keepalive(true); + _input = s.input(); + _output = s.output(); + }); + }); + }); +} + +future<> kmip_host::impl::connection::wait_for_io() { + kmip_log.trace("{}: Waiting...", *this); + auto o = std::exchange(_pending, std::nullopt); + return o ? std::move(*o) : make_ready_future(); +} + +int kmip_host::impl::connection::send(void* data, unsigned int len, unsigned int*) { + if (_pending) { + kmip_log.trace("{}: operation pending...", *this); + return KMIP_ERROR_RETRY; + } + kmip_log.trace("{}: Sending {} bytes", *this, len); + + auto f = _output.write(reinterpret_cast(data), len).then([this] { + kmip_log.trace("{}: send done. flushing...", *this); + return _output.flush(); + }); + // if the call failed already, we still want to + // drop back to "wait_for_io()", because we cannot throw + // exceptions through the kmipc code frames. + if (!f.available() || f.failed()) { + _pending.emplace(std::move(f)); + } + return KMIP_ERROR_NONE; +} + +int kmip_host::impl::connection::recv(void* data, unsigned int len, unsigned int* outlen) { + kmip_log.trace("{}: Waiting for data ({})", *this, len); + for (;;) { + if (_in_buffer) { + auto n = std::min(unsigned(_in_buffer->size()), len); + *outlen = n; + kmip_log.trace("{}: returning {} ({}) bytes", *this, n, _in_buffer->size()); + std::copy(_in_buffer->begin(), _in_buffer->begin() + n, reinterpret_cast(data)); + _in_buffer->trim_front(n); + if (_in_buffer->empty()) { + _in_buffer = std::nullopt; + } + // #998 cryptsoft example returns error on EOF. + if (n == 0) { + return KMIP_ERROR_IO; + } + break; + } + + if (_pending) { + kmip_log.trace("{}: operation pending...", *this); + return KMIP_ERROR_RETRY; + } + + kmip_log.trace("{}: issue read", *this); + auto f = _input.read().then([this](temporary_buffer buf) { + kmip_log.trace("{}: got {} bytes", *this, buf.size()); + _in_buffer = std::move(buf); + }); + + // if the call failed already, we still want to + // drop back to "wait_for_io()", because we cannot throw + // exceptions through the kmipc code frames. + if (!f.available() || f.failed()) { + _pending.emplace(std::move(f)); + } + } + return KMIP_ERROR_NONE; +} + +int kmip_host::impl::connection::io_callback(KMIP *kmip, void *cb_arg, int op, void *data, unsigned int len, unsigned int *outlen) { + auto* conn = reinterpret_cast(cb_arg); + try { + switch(op) { + default: + return KMIP_ERROR_NOT_SUPPORTED; + case KMIP_IO_CMD_SEND: + return conn->send(data, len, outlen); + case KMIP_IO_CMD_RECV: + return conn->recv(data, len, outlen); + } + } catch (...) { + kmip_log.warn("Error in KMIP IO: {}", std::current_exception()); + return KMIP_ERROR_IO; + } +} + +void kmip_host::impl::connection::attach(KMIP_CMD* cmd) { + kmip_log.trace("{} Attach: {}", *this, reinterpret_cast(cmd)); + if (cmd == nullptr) { + return; + } + + if (!_options.username.empty()) { + kmip_chk( + KMIP_CMD_set_credential_username(cmd, + const_cast(_options.username.c_str()), + const_cast(_options.password.c_str()))); + } + + /* because we haven't passed in anything to the KMIP_CMD layer + * that would provide it with the protocol version details we + * have to separately indicate that here + */ + kmip_chk(KMIP_CMD_set_lib_protocol(cmd, KMIP_LIB_PROTOCOL_KMIP1)); + /* handle all IO via the callback */ + kmip_chk( + KMIP_CMD_set_io_cb(cmd, &connection::io_callback, + reinterpret_cast(this))); +} + +future<> kmip_host::impl::connection::close() { + return _output.close().finally([this] { + return _input.close(); + }); +} + +template +class kmip_host::impl::kmip_handle { +public: + kmip_handle(T * ptr) + : _ptr(ptr, FreeFunc) + {} + kmip_handle(kmip_handle&&) = default; + kmip_handle& operator=(kmip_handle&&) = default; + + T* get() const { + return _ptr.get(); + } + operator T*() const { + return _ptr.get(); + } + explicit operator bool() const { + return _ptr != nullptr; + } +private: + using ptr_type = std::unique_ptr; + ptr_type _ptr; +}; + +class kmip_host::impl::kmip_cmd : public kmip_handle { +public: + kmip_cmd(int flags = KMIP_CMD_FLAGS_DEFAULT|KMIP_CMD_FLAGS_LOG|KMIP_CMD_FLAGS_LOG_XML) + : kmip_handle([flags] { + KMIP_CMD* cmd; + kmip_chk(KMIP_CMD_new_ex(flags, nullptr, &cmd)); + return cmd; + }()) + {} + kmip_cmd(kmip_cmd&&) = default; + kmip_cmd& operator=(kmip_cmd&&) = default; + + friend std::ostream& operator<<(std::ostream& os, const kmip_cmd& cmd) { + return os << KMIP_CMD_get_request(cmd); + } +}; + +} + +template <> struct fmt::formatter : fmt::ostream_formatter {}; + +namespace encryption { + +class kmip_host::impl::kmip_data_list : public kmip_handle { +public: + kmip_data_list(int flags = KMIP_DATA_LIST_FLAGS_DEFAULT) + : kmip_handle([flags] { + KMIP_DATA_LIST* kdl; + kmip_chk(KMIP_DATA_LIST_new(flags, &kdl)); + return kdl; + }()) + {} + kmip_data_list(kmip_data_list&&) = default; + kmip_data_list& operator=(kmip_data_list&&) = default; +}; + +/** + * Clears and releases a connection cp. Release connection after. + * If retain_connection is true, the connection is only cleared of command data and + * can be reused by caller, otherwise it is either added to the connection pool + * or dropped. +*/ +void kmip_host::impl::release(KMIP_CMD* cmd, con_ptr cp, bool retain_connection) { + auto i = _host_connections.find(cp->host()); + userdata u; + u.host = i->first.c_str(); + if (cmd) { + KMIP_CMD_set_userdata(cmd, u.ptr); + } + if (!retain_connection && is_current_host(i->first) && max_pooled_connections_per_host() > i->second.size()) { + i->second.emplace_back(std::move(cp)); + } +} + +/** + * Run a function on a KMIP command using connection cp. Release connection after. + * If retain_connection_after_command is true, the connection is only cleared of command data and + * can be reused by caller. +*/ +template +future kmip_host::impl::do_cmd(KMIP_CMD* cmd, con_ptr cp, Func& f, bool retain_connection_after_command) { + cp->attach(cmd); + + return repeat_until_value([this, cmd, &f, cp, retain_connection_after_command] { + int res = f(cmd); + switch (res) { + case KMIP_ERROR_RETRY: + return cp->wait_for_io().then([] { + return opt_int(); + }).handle_exception([cp](auto ep) { + // get here if we had any wire exceptions below. + // make sure to force flush and stuff here as well. + return cp->close().then_wrapped([ep = std::move(ep)](auto f) mutable { + try { + f.get(); + } catch (...) { + } + return make_exception_future(std::move(ep)); + }); + }); + case 0: + release(cmd, cp, retain_connection_after_command); + return make_ready_future(res); + default: + // error. connection is dicarded. close it. + return cp->close().then_wrapped([cp, res](auto f) { + // ignore any exception thrown from the close. + // ensure we provide the kmip error instead. + try { + f.get(); + } catch (...) { + } + return make_ready_future(res); + }); + } + }).finally([cp] {}); +} + +template +future kmip_host::impl::do_cmd(kmip_cmd cmd_in, Func && f) { + kmip_log.trace("{}: begin do_cmd", *this, cmd_in); + KMIP_CMD* cmd = cmd_in; + + // #998 Need to do retry loop, because we can have either timed out connection, + // lost it (connected server went down) or some other network error. + return do_with(std::move(f), [this, cmd](Func& f) { + return repeat_until_value([this, cmd, &f, retry = _max_retry]() mutable { + --retry; + return get_connection(cmd).handle_exception([this, cmd, retry](std::exception_ptr ep) { + if (retry) { + // failed to connect. do more serious backing off. + // we only retry this once, since get_connection + // will either give back cached connections, + // or explicitly try all avail hosts. + // In the first case, we will do the lower retry + // loop if something is stale/borked, the latter is + // more or less dead. + auto sleeptime = base_backoff_time * (_max_retry - retry); + kmip_log.debug("{}: Connection failed. backoff {}", *this, std::chrono::duration_cast(sleeptime).count()); + return seastar::sleep(sleeptime).then([this, cmd] { + kmip_log.debug("{}: retrying...", *this); + return get_connection(cmd); + }); + } + return make_exception_future(std::move(ep)); + }).then([this, cmd, &f, retry](con_ptr cp) mutable { + auto host = cp->host(); + auto res = do_cmd(cmd, std::move(cp), f); + kmip_log.trace("{}: request {}", *this, fmt::ptr(KMIP_CMD_get_request(cmd))); + return res.then([this, retry, host = std::move(host)](int res) { + if (res == KMIP_ERROR_IO) { + kmip_log.debug("{}: request error {}", *this, kmip_errorc.message(res)); + if (retry) { + // do some backing off unless this is the first retry, which + // might be a stale connection. Clear out all caches for the + // current host first, then retry. + auto f = clear_connections(host); + if (retry != (_max_retry - 1)) { + f = f.then([this] { + auto sleeptime = base_backoff_time; + kmip_log.debug("{}: backoff {}ms", *this, std::chrono::duration_cast(sleeptime).count()); + return seastar::sleep(sleeptime); + }); + } + return f.then([this] { + kmip_log.debug("{}: retrying...", *this); + return opt_int{}; + }); + } + } + return make_ready_future(res); + }); + }); + }); + }).then([this, cmd = std::move(cmd_in)](int res) mutable { + kmip_chk(res, cmd); + kmip_log.trace("{}: result {}", *this, fmt::ptr(KMIP_CMD_get_response(cmd))); + return std::move(cmd); + }); +} + +future kmip_host::impl::get_connection(const sstring& host) { + // TODO: if a pooled connection is stale, the command run will fail, + // and the connection will be discarded. Would be good if we could detect this case + // and re-run command with a new connection. Maybe always verify connection, even if + // it is old? + auto& q = _host_connections[host]; + + if (!q.empty()) { + auto cp = q.front(); + q.pop_front(); + return make_ready_future<::shared_ptr>(cp); + } + + auto cp = ::make_shared(host, _options); + kmip_log.trace("{}: connecting to {}", *this, host); + return cp->connect().then([this, cp, host] { + kmip_log.trace("{}: verifying {}", *this, host); + kmip_cmd cmd; + static auto connection_query = [](KMIP_CMD* cmd) { + static const std::array query_options = { + KMIP_QUERY_FUNCTION_QUERY_OPERATIONS, + KMIP_QUERY_FUNCTION_QUERY_OBJECTS, + }; + return KMIP_CMD_query(cmd, const_cast(query_options.data()), unsigned(query_options.size())); + }; + // when/if this succeeds, it will push the connection onto the available stack + auto f = do_cmd(cmd, cp, connection_query, true /* keep cp */); + return f.then([this, host, cmd = std::move(cmd), cp](int res) { + kmip_chk(res, cmd); + kmip_log.trace("{}: connected {}", *this, host); + return cp; + }); + }); +} + + +future kmip_host::impl::get_connection(KMIP_CMD* cmd) { + userdata u{ KMIP_CMD_get_userdata(cmd) }; + if (u.host != nullptr) { + return get_connection(u.host).then([](con_ptr cp) { + return cp; + }); + } + + using con_ptr = ::shared_ptr; + using con_opt = std::optional; + + return repeat_until_value([this, i = size_t(0)]() mutable { + if (i++ == _options.hosts.size()) { + throw missing_resource_error("Could not connect to any server"); + } + auto& host = _options.hosts[_index % _options.hosts.size()]; + return get_connection(host).then([](con_ptr cp) { + return con_opt(std::move(cp)); + }).handle_exception([this, host](auto) { + ++_index; + // if we fail one host, clear out any + // caches for it just in case. + return clear_connections(host).then([] { + return con_opt(); + }); + }); + }); +} + +future<> kmip_host::impl::clear_connections(const sstring& host) { + auto q = std::exchange(_host_connections[host], {}); + return parallel_for_each(q.begin(), q.end(), [](con_ptr c) { + return c->close().handle_exception([c](auto ep) { + // ignore exceptions + }); + }); +} + +future<> kmip_host::impl::connect() { + return do_for_each(_options.hosts, [this](const sstring& host) { + return get_connection(host).then([this](auto cp) { + release(nullptr, cp); + }); + }); +} + +future<> kmip_host::impl::disconnect() { + return do_for_each(_options.hosts, [this](const sstring& host) { + return clear_connections(host); + }); +} + +static unsigned from_str(unsigned (*f)(char*, int, int*), const sstring& s, const sstring& what) { + int found = 0; + auto res = f(const_cast(s.c_str()), CODE2STR_FLAG_STR_CASE, &found); + if (!found) { + throw std::invalid_argument(format("Unsupported {}: {}", what, s)); + } + return res; +} + +std::tuple kmip_host::impl::make_attributes(const kmip_key_info& info, bool include_template) const { + kmip_data_list kdl_attrs; + + if (!info.options.template_name.empty()) { + kmip_chk(KMIP_DATA_LIST_add_attr_str_by_tag(kdl_attrs, + KMIP_TAG_TEMPLATE, + const_cast(info.options.template_name.c_str())) + ); + } + if (!info.options.key_namespace.empty()) { + kmip_chk(KMIP_DATA_LIST_add_attr_str(kdl_attrs, + const_cast("x-key-namespace"), + const_cast(info.options.key_namespace.c_str())) + ); + } + sstring type, mode, padd; + std::tie(type, mode, padd) = parse_key_spec_and_validate_defaults(info.info.alg); + + try { + auto crypt_alg = from_str(&KMIP_string_to_CRYPTOGRAPHIC_ALGORITHM, type, "cryptographic algorithm"); + return std::make_tuple(std::move(kdl_attrs), crypt_alg); + } catch (std::invalid_argument& e) { + std::throw_with_nested(std::invalid_argument("Invalid algorithm: " + info.info.alg)); + } +} + +kmip_host::id_type kmip_host::impl::kmip_id_to_id(const sstring& s) const { + try { + // #2205 - we previously made all ID:s into uuids (because the literal functions + // are called KMIP_CMD_get_uuid etc). This has issues with Keysecure which apparently + // does _not_ give back UUID format strings, but "other" things. + // Could just always store ascii as bytes instead, but that would now + // break existing installations, so we check for UUID, and if it does not + // match we encode it. + utils::UUID uuid(s); + return uuid.serialize(); + } catch (marshal_exception&) { + // very simple exncoding scheme: add a "len" byte at the end. + // iff byte size of id + 1 (len) equals 16 (length of UUID), + // add a padding byte. + size_t len = s.size() + 1; + if (len == 16) { + ++len; + } + bytes res(len, 0); + std::copy(s.begin(), s.end(), res.begin()); + res.back() = int8_t(len - s.size()); + return res; + } +} + +sstring kmip_host::impl::id_to_kmip_string(const id_type& id) const { + // see comment above for encoding scheme. + if (id.size() == 16) { + // if byte size is UUID it must be a UUID. No "old" id:s are + // not, and we never encode non-uuid as 16 bytes. + auto uuid = utils::UUID_gen::get_UUID(id); + return fmt::format("{}", uuid); + } + auto len = id.size() - id.back(); + return sstring(id.begin(), id.begin() + len); +} + +future kmip_host::impl::create_key(const kmip_key_info& info) { + if (this_shard_id() == 0) { + // #1039 First try looking for existing keys on server + return find_matching_keys(info, 1).then([this, info](std::vector ids) { + if (!ids.empty()) { + // got it + return get_key_by_id(ids.front(), info.info).then([id = ids.front()](shared_ptr k) { + return key_and_id_type(std::move(k), id); + }); + } + + kmip_log.debug("{}: Creating key {}", _name, info); + + auto kdl_attrs_crypt_alg = make_attributes(info); + auto&& kdl_attrs = std::get<0>(kdl_attrs_crypt_alg); + auto&& crypt_alg = std::get<1>(kdl_attrs_crypt_alg); + + // TODO: this is inefficient. We can probably put this in a single batch. + kmip_cmd cmd; + KMIP_CMD_set_ctx(cmd, const_cast("Create key")); + + return do_cmd(std::move(cmd), [info, kdl_attrs = std::move(kdl_attrs), crypt_alg](KMIP_CMD* cmd) { + return KMIP_CMD_create_smpl(cmd, KMIP_OBJECT_TYPE_SYMMETRIC_KEY, + crypt_alg, + KMIP_CRYPTOGRAPHIC_USAGE_ENCRYPT|KMIP_CRYPTOGRAPHIC_USAGE_DECRYPT, + int(info.info.len), + KMIP_DATA_LIST_attrs(kdl_attrs), KMIP_DATA_LIST_n_attrs(kdl_attrs) + ); + }).then([this, info](kmip_cmd cmd) { + /* now get the details (the value of the key) */ + char* new_id; + kmip_chk(KMIP_CMD_get_uuid(cmd, 0, &new_id), cmd); + sstring uuid(new_id); + + kmip_log.debug("{}: Created {}:{}", _name, info, uuid); + + KMIP_CMD_set_ctx(cmd, const_cast("activate")); + + return do_cmd(std::move(cmd), [new_id](KMIP_CMD* cmd) { + return KMIP_CMD_activate(cmd, new_id); + }).then([this, info, uuid](kmip_cmd cmd) { + auto id = kmip_id_to_id(uuid); + kmip_log.debug("{}: Activated {}", _name, uuid); + return get_key_by_id(id, info.info).then([id](auto k) { + return key_and_id_type(k, id); + }); + }); + }); + }); + } + + return smp::submit_to(0, [this, info] { + return _ctxt.get_kmip_host(_name)->get_or_create_key(info.info, info.options).then([](std::tuple, id_type> k_id) { + auto&& [k, id] = k_id; + return make_ready_future>(std::tuple(k->info(), k->key(), id)); + }); + }).then([](std::tuple info_b_id) { + auto&& [info, b, id] = info_b_id; + return make_ready_future(key_and_id_type(make_shared(info, b), id)); + }); +} + +future> kmip_host::impl::find_matching_keys(const kmip_key_info& info, std::optional max) { + kmip_log.debug("{}: Finding matching key {}", _name, info); + + auto [kdl_attrs, crypt_alg] = make_attributes(info, false); + + static const char kmip_tag_cryptographic_length[] = KMIP_TAG_CRYPTOGRAPHIC_LENGTH_STR; + static const char kmip_tag_cryptographic_usage_mask[] = KMIP_TAG_CRYPTOGRAPHIC_USAGE_MASK_STR; + + // #1079. Query mask apparently ignores things like cryptographic + // attribute set of options, instead we must specify the query + // as a list of attributes. + kmip_chk(KMIP_DATA_LIST_add_attr_enum_by_tag(kdl_attrs, + KMIP_TAG_OBJECT_TYPE, + KMIP_OBJECT_TYPE_SYMMETRIC_KEY) + ); + kmip_chk(KMIP_DATA_LIST_add_attr_enum_by_tag(kdl_attrs, + KMIP_TAG_CRYPTOGRAPHIC_ALGORITHM, + int(crypt_alg)) + ); + kmip_chk(KMIP_DATA_LIST_add_attr_int(kdl_attrs, + // our kmip sdk is broken/const-challenged + const_cast(kmip_tag_cryptographic_length), + int(info.info.len)) + ); + kmip_chk(KMIP_DATA_LIST_add_attr_enum_by_tag(kdl_attrs, + KMIP_TAG_STATE, + KMIP_STATE_ACTIVE) + ); + kmip_chk(KMIP_DATA_LIST_add_attr_int(kdl_attrs, + const_cast(kmip_tag_cryptographic_usage_mask), + KMIP_CRYPTOGRAPHIC_USAGE_ENCRYPT|KMIP_CRYPTOGRAPHIC_USAGE_DECRYPT) + ); + + kmip_cmd cmd; + KMIP_CMD_set_ctx(cmd, const_cast("Find matching key")); + + std::unique_ptr mp; + int* maxp = nullptr; + if (max) { + mp = std::make_unique(*max); + maxp = mp.get(); + } + + return do_cmd(std::move(cmd), [kdl_attrs = std::move(kdl_attrs), maxp](KMIP_CMD* cmd) { + return KMIP_CMD_locate(cmd, maxp, nullptr, KMIP_DATA_LIST_attrs(kdl_attrs), KMIP_DATA_LIST_n_attrs(kdl_attrs)); + }).then([this, info, mp = std::move(mp)](kmip_cmd cmd) { + std::vector result; + + for (int i = 0; ; ++i) { + char* new_id; + auto err = KMIP_CMD_get_uuid(cmd, i, &new_id); + if (err == KMIP_ERROR_NOT_FOUND) { + break; + } + kmip_chk(err, cmd); + result.emplace_back(kmip_id_to_id(new_id)); + } + + kmip_log.debug("{}: Found {} matching keys {}", _name, result.size(), info); + + return result; + }); +} + +future> kmip_host::impl::find_key(const id_type& id) { + if (this_shard_id() == 0) { + kmip_cmd cmd; + KMIP_CMD_set_ctx(cmd, const_cast("Find key")); + + auto uuid = id_to_kmip_string(id); + kmip_log.debug("{}: Finding {}", _name, uuid); + + // Batch operation. Nothing is sent/received until xmit below + kmip_chk(KMIP_CMD_batch_start(cmd)); + kmip_chk(KMIP_CMD_set_batch_order(cmd, 1)); + { + int key_format_type = KMIP_KEY_FORMAT_TYPE_RAW; + kmip_chk(KMIP_CMD_get(cmd, const_cast(uuid.c_str()), &key_format_type, nullptr, nullptr)); + } + kmip_chk(KMIP_CMD_get_attributes(cmd, const_cast(uuid.c_str()), nullptr, 0)); + + return do_cmd(std::move(cmd), [](KMIP_CMD* cmd) { + return KMIP_CMD_batch_xmit(cmd); + }).then([this, uuid](kmip_cmd cmd) { + auto nb = KMIP_CMD_get_batch_count(cmd); + if (nb != 2) { + throw malformed_response_error("Invalid batch count in response: " + std::to_string(nb)); + } + + sstring alg; + sstring mode; + sstring padd; + + // "Get" result + auto kdl_res = KMIP_CMD_get_batch(cmd, 0); + + /* get a reference to the key material (the actual key value) */ + unsigned char* key; + unsigned int keylen; + kmip_chk(KMIP_DATA_LIST_get_data(kdl_res, KMIP_TAG_KEY_MATERIAL, 0, &key, &keylen)); + + auto tag_to_string = [](auto f, auto val) { + int found; + auto p = f(val, CODE2STR_FLAG_STR_CASE, &found); + if (!found) { + throw malformed_response_error("Invalid tag: " + std::to_string(val)); + } + return sstring(p); + }; + + int crypto_alg; + kmip_chk(KMIP_DATA_LIST_get_32(kdl_res, KMIP_TAG_CRYPTOGRAPHIC_ALGORITHM, 0, &crypto_alg)); + alg = tag_to_string(&KMIP_CRYPTOGRAPHIC_ALGORITHM_to_string, crypto_alg); + + // "Attribute list" result + // This will apparently most of the time _not_ contain the info we want, + // depending on server, but we record as much as we can anyway. + // The actual resulting keys used will be based on external config. Only + // key data and verifying that it is compatible with said info is + // important for us. + auto kdl_attr = KMIP_CMD_get_batch(cmd, 1); + + unsigned int attr_count = 0; + kmip_chk(KMIP_DATA_LIST_get_count(kdl_attr, KMIP_TAG_ATTRIBUTE, &attr_count)); + + for (unsigned int i = 0; i < attr_count; i++) { + KMIP_DATA *attr = nullptr; + int n_attr = 0; + + kmip_chk(KMIP_DATA_LIST_get_struct(kdl_attr, KMIP_TAG_ATTRIBUTE, i, &attr, &n_attr, NULL)); + + + KMIP_DATA *attr_val = nullptr; + kmip_chk(KMIP_DATA_get(attr, n_attr,KMIP_TAG_ATTRIBUTE_VALUE, 0, &attr_val)); + + switch (attr_val->tag) { + case KMIP_TAG_BLOCK_CIPHER_MODE: + mode = tag_to_string(&KMIP_BLOCK_CIPHER_MODE_to_string, attr_val->data32); + break; + case KMIP_TAG_PADDING_METHOD: + padd = tag_to_string(&KMIP_PADDING_METHOD_to_string, attr_val->data32); + break; + default: + break; + } + } + + if (alg.empty()) { + throw configuration_error("Could not find algorithm"); + } + if (mode.empty() != padd.empty()) { + throw configuration_error("Invalid block mode/padding"); + } + + auto str = mode.empty() || padd.empty() ? alg : alg + "/" + mode + "/" + padd; + key_info derived_info{ str, keylen*8}; + + kmip_log.trace("{}: Found {}:{} {}", _name, uuid, derived_info.alg, derived_info.len); + + return make_shared(derived_info, bytes(key, key + keylen)); + }); + } + + return smp::submit_to(0, [this, id] { + return _ctxt.get_kmip_host(_name)->get_key_by_id(id).then([](shared_ptr k) { + return make_ready_future>(std::tuple(k->info(), k->key())); + }); + }).then([](std::tuple info_b) { + auto&& [info, b] = info_b; + return make_shared(info, b); + }); +} + +shared_ptr kmip_host::impl::ensure_compatible_key(shared_ptr k, const key_info& info) { + // keys we get back are typically void + // of block mode/padding info (because this is meaningless + // from the standpoint of the kmip server). + // Check and re-init the actual key used based + // on what the user wants so we adhere to block mode etc. + if (!info.compatible(k->info())) { + throw malformed_response_error(fmt::format("Incompatible key: {}", k->info())); + } + if (k->info() != info) { + k = ::make_shared(info, k->key()); + } + return k; +} + +[[noreturn]] +static void translate_kmip_error(const kmip_error& e) { + switch (e.code().value()) { + case KMIP_ERROR_BAD_CONNECT: case KMIP_ERROR_IO: + std::throw_with_nested(network_error(e.what())); + case KMIP_ERROR_BAD_PROTOCOL: + std::throw_with_nested(configuration_error(e.what())); + case KMIP_ERROR_NOT_FOUND: + std::throw_with_nested(missing_resource_error(e.what())); + case KMIP_ERROR_AUTH_FAILED: case KMIP_ERROR_CERT_AUTH_FAILED: + std::throw_with_nested(permission_error(e.what())); + default: + std::throw_with_nested(service_error(e.what())); + } +} + +future, kmip_host::id_type>> kmip_host::impl::get_or_create_key(const key_info& info, const key_options& opts) { + kmip_log.debug("{}: Lookup key {}:{}", _name, info, opts); + try { + auto linfo = info; + auto kinfo = co_await _attr_cache.get(kmip_key_info{info, opts}); + co_return std::tuple(ensure_compatible_key(std::get<0>(kinfo), linfo), std::get<1>(kinfo)); + } catch (kmip_error& e) { + translate_kmip_error(e); + } catch (base_error&) { + throw; + } catch (std::invalid_argument& e) { + std::throw_with_nested(configuration_error(fmt::format("get_or_create_key: {}", e.what()))); + } catch (...) { + std::throw_with_nested(service_error(fmt::format("get_or_create_key: {}", std::current_exception()))); + } +} + +future> kmip_host::impl::get_key_by_id(const id_type& id, const std::optional& info) { + try { + auto linfo = info; // maintain on stack + auto k = co_await _id_cache.get(id); + if (linfo) { + k = ensure_compatible_key(k, *linfo); + } + co_return k; + } catch (kmip_error& e) { + translate_kmip_error(e); + } catch (base_error&) { + throw; + } catch (std::invalid_argument& e) { + std::throw_with_nested(configuration_error(fmt::format("get_key_by_id: {}", e.what()))); + } catch (...) { + std::throw_with_nested(service_error(fmt::format("get_key_by_id: {}", std::current_exception()))); + } +} + +kmip_host::kmip_host(encryption_context& ctxt, const sstring& name, const std::unordered_map& map) + : kmip_host(ctxt, name, [&ctxt, &map] { + host_options opts; + map_wrapper> m(map); + + try { + static const std::regex wsc("\\s*,\\s*"); // comma+whitespace + + std::string hosts = m("hosts").value(); + + auto i = std::sregex_token_iterator(hosts.begin(), hosts.end(), wsc, -1); + auto e = std::sregex_token_iterator(); + + std::for_each(i, e, [&](const std::string & s) { + opts.hosts.emplace_back(s); + }); + } catch (std::bad_optional_access&) { + throw std::invalid_argument("No KMIP host names provided"); + } + + opts.certfile = m("certificate").value_or(""); + opts.keyfile = m("keyfile").value_or(""); + opts.truststore = m("truststore").value_or(""); + opts.priority_string = m("priority_string").value_or(""); + + opts.username = m("username").value_or(""); + opts.password = ctxt.maybe_decrypt_config_value(m("password").value_or("")); + + if (m("max_command_retries")) { + opts.max_command_retries = std::stoul(*m("max_command_retries")); + } + + opts.key_cache_expiry = parse_expiry(m("key_cache_expiry")); + opts.key_cache_refresh = parse_expiry(m("key_cache_refresh")); + + return opts; + }()) +{} + +kmip_host::kmip_host(encryption_context& ctxt, const sstring& name, const host_options& opts) + : _impl(std::make_unique(ctxt, name, opts)) +{} + +kmip_host::~kmip_host() = default; + +future<> kmip_host::connect() { + return _impl->connect(); +} + +future<> kmip_host::disconnect() { + return _impl->disconnect(); +} + +future, kmip_host::id_type>> kmip_host::get_or_create_key(const key_info& info, const key_options& opts) { + return _impl->get_or_create_key(info, opts); +} + +future> kmip_host::get_key_by_id(const id_type& id, std::optional info) { + return _impl->get_key_by_id(id, info); +} + +future> kmip_host::get_key_by_name(const sstring& name) { + return _impl->get_key_by_id(_impl->kmip_id_to_id(name)); +} + +std::ostream& operator<<(std::ostream& os, const kmip_host::key_options& opts) { + return os << opts.template_name << ":" << opts.key_namespace; +} + +} + +#else + +#include "kmip_host.hh" + +namespace encryption { + +class kmip_host::impl { +}; + +kmip_host::kmip_host(encryption_context& ctxt, const sstring& name, const std::unordered_map& map) { + throw std::runtime_error("KMIP support not enabled"); +} + +kmip_host::kmip_host(encryption_context& ctxt, const sstring& name, const host_options& opts) { + throw std::runtime_error("KMIP support not enabled"); +} + +kmip_host::~kmip_host() = default; + +future<> kmip_host::connect() { + throw std::runtime_error("KMIP support not enabled"); +} + +future<> kmip_host::disconnect() { + throw std::runtime_error("KMIP support not enabled"); +} + +future, kmip_host::id_type>> kmip_host::get_or_create_key(const key_info& info, const key_options& opts) { + throw std::runtime_error("KMIP support not enabled"); +} + +future> kmip_host::get_key_by_id(const id_type& id, std::optional info) { + throw std::runtime_error("KMIP support not enabled"); +} + +future> kmip_host::get_key_by_name(const sstring& name) { + throw std::runtime_error("KMIP support not enabled"); +} + +std::ostream& operator<<(std::ostream& os, const kmip_host::key_options& opts) { + return os << opts.template_name << ":" << opts.key_namespace; +} + +} + +#endif diff --git a/ent/encryption/kmip_host.hh b/ent/encryption/kmip_host.hh new file mode 100644 index 0000000000..05963e2784 --- /dev/null +++ b/ent/encryption/kmip_host.hh @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "../../bytes.hh" + +#include "symmetric_key.hh" + +namespace encryption { + +class symmetric_key; +class encryption_context; +struct key_info; + +class kmip_host { +public: + struct host_options { + std::vector hosts; + + sstring username; + sstring password; + + sstring certfile; + sstring keyfile; + sstring truststore; + sstring priority_string; + + std::optional key_cache_expiry; + std::optional key_cache_refresh; + + std::optional max_pooled_connections_per_host; + std::optional max_command_retries; + }; + struct key_options { + sstring template_name; + sstring key_namespace; + }; + using id_type = bytes; + + kmip_host(encryption_context&, const sstring& name, const host_options&); + kmip_host(encryption_context&, const sstring& name, const std::unordered_map&); + ~kmip_host(); + + future<> connect(); + future<> disconnect(); + future, id_type>> get_or_create_key(const key_info&, const key_options& = {}); + future> get_key_by_id(const id_type&, std::optional = std::nullopt); + + /** for system key(s) */ + future> get_key_by_name(const sstring&); + +private: + class impl; + std::unique_ptr _impl; +}; + +std::ostream& operator<<(std::ostream&, const kmip_host::key_options&); + +} + +template <> struct fmt::formatter : fmt::ostream_formatter {}; diff --git a/ent/encryption/kmip_key_provider.cc b/ent/encryption/kmip_key_provider.cc new file mode 100644 index 0000000000..6d98a217c0 --- /dev/null +++ b/ent/encryption/kmip_key_provider.cc @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#include +#include + +#include "utils/UUID.hh" +#include "utils/UUID_gen.hh" + +#include "kmip_key_provider.hh" +#include "kmip_host.hh" + +namespace encryption { + +class kmip_key_provider : public key_provider { +public: + kmip_key_provider(::shared_ptr kmip_host, kmip_host::key_options kopts, sstring name) + : _kmip_host(std::move(kmip_host)) + , _kopts(std::move(kopts)) + , _name(std::move(name)) + {} + future> key(const key_info& info, opt_bytes id) override { + if (id) { + return _kmip_host->get_key_by_id(*id, info).then([id](key_ptr k) { + return make_ready_future>(std::tuple(k, id)); + }); + } + return _kmip_host->get_or_create_key(info, _kopts).then([](std::tuple k_id) { + return make_ready_future>(k_id); + }); + } + void print(std::ostream& os) const override { + os << _name; + if (!_kopts.key_namespace.empty()) { + os << ", namespace=" << _kopts.key_namespace; + } + if (!_kopts.template_name.empty()) { + os << ", template=" << _kopts.template_name; + } + } + +private: + ::shared_ptr _kmip_host; + kmip_host::key_options _kopts; + sstring _name; +}; + + +shared_ptr kmip_key_provider_factory::get_provider(encryption_context& ctxt, const options& map) { + opt_wrapper opts(map); + auto host = opts(HOST_NAME); + if (!host) { + throw std::invalid_argument("kmip_host must be provided"); + } + kmip_host::key_options kopts = { + opts(TEMPLATE_NAME).value_or(""), + opts(KEY_NAMESPACE).value_or(""), + }; + + auto cache_key = *host + ":" + boost::lexical_cast(kopts); + auto provider = ctxt.get_cached_provider(cache_key); + + if (!provider) { + provider = ::make_shared(ctxt.get_kmip_host(*host), std::move(kopts), *host); + ctxt.cache_provider(cache_key, provider); + } + + return provider; +} + +static std::optional> parse_kmip_host_and_path(const sstring & s) { + static const std::regex kmip_ex("kmip://([^/]+)/([\\w/]+)"); + + std::match_results m; + if (std::regex_match(s.begin(), s.end(), m, kmip_ex)) { + return std::make_pair(sstring(m[1]), sstring(m[2])); + } + return std::nullopt; +} + +kmip_system_key::kmip_system_key(encryption_context& ctxt, const sstring& s) { + auto p = parse_kmip_host_and_path(s); + if (!p) { + throw std::invalid_argument("Not a kmip path: " + s); + } + + _host = ctxt.get_kmip_host(p->first); + _name = p->second; +} + +kmip_system_key::~kmip_system_key() = default; + +bool kmip_system_key::is_kmip_path(const sstring& s) { + return parse_kmip_host_and_path(s) != std::nullopt; +} + +future> kmip_system_key::get_key() { + if (_key) { + return make_ready_future>(_key); + } + return _host->get_key_by_name(_name).then([this](shared_ptr k) { + _key = k; + return k; + }); +} + +const sstring& kmip_system_key::name() const { + return _name; +} + + +} + diff --git a/ent/encryption/kmip_key_provider.hh b/ent/encryption/kmip_key_provider.hh new file mode 100644 index 0000000000..c383b7fb0f --- /dev/null +++ b/ent/encryption/kmip_key_provider.hh @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ +#pragma once + +#include "encryption.hh" +#include "system_key.hh" + +namespace encryption { + +class kmip_key_provider_factory : public key_provider_factory { +public: + shared_ptr get_provider(encryption_context&, const options&) override; +}; + +class kmip_host; + +class kmip_system_key : public system_key { + shared_ptr _key; + shared_ptr _host; + sstring _name; +public: + kmip_system_key(encryption_context&, const sstring&); + ~kmip_system_key(); + + static bool is_kmip_path(const sstring&); + + future> get_key() override; + const sstring& name() const override; + bool is_local() const override { + return false; + } +}; + +} diff --git a/ent/encryption/kms_host.cc b/ent/encryption/kms_host.cc new file mode 100644 index 0000000000..2827f54efa --- /dev/null +++ b/ent/encryption/kms_host.cc @@ -0,0 +1,1164 @@ +/* + * Copyright (C) 2022 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include "utils/to_string.hh" + +#include "kms_host.hh" +#include "encryption.hh" +#include "encryption_exceptions.hh" +#include "symmetric_key.hh" +#include "utils/hash.hh" +#include "utils/loading_cache.hh" +#include "utils/UUID.hh" +#include "utils/UUID_gen.hh" +#include "utils/rjson.hh" +#include "marshal_exception.hh" +#include "db/config.hh" + +template struct fmt::formatter> : fmt::ostream_formatter {}; +template <> struct fmt::formatter : fmt::ostream_formatter {}; + +using namespace std::chrono_literals; +using namespace std::string_literals; + +logger kms_log("kms"); + +class kms_error : public std::exception { + std::string _type, _msg; +public: + kms_error(std::string_view type, std::string_view msg) + : _type(type) + , _msg(fmt::format("{}: {}", type, msg)) + {} + const std::string& type() const { + return _type; + } + const char* what() const noexcept override { + return _msg.c_str(); + } +}; + +namespace kms_errors { + [[maybe_unused]] static const char* AccessDeniedException = "AccessDeniedException"; + [[maybe_unused]] static const char* IncompleteSignature = "IncompleteSignature"; + [[maybe_unused]] static const char* InternalFailure = "InternalFailure"; + [[maybe_unused]] static const char* InvalidAction = "InvalidAction"; + [[maybe_unused]] static const char* InvalidClientTokenId = "InvalidClientTokenId"; + [[maybe_unused]] static const char* InvalidParameterCombination = "InvalidParameterCombination"; + [[maybe_unused]] static const char* InvalidParameterValue = "InvalidParameterValue"; + [[maybe_unused]] static const char* InvalidQueryParameter = "InvalidQueryParameter"; + [[maybe_unused]] static const char* MalformedQueryString = "MalformedQueryString"; + [[maybe_unused]] static const char* MissingAction = "MissingAction"; + [[maybe_unused]] static const char* MissingAuthenticationToken = "MissingAuthenticationToken"; + [[maybe_unused]] static const char* MissingParameter = "MissingParameter"; + [[maybe_unused]] static const char* NotAuthorized = "NotAuthorized"; + [[maybe_unused]] static const char* OptInRequired = "OptInRequired"; + [[maybe_unused]] static const char* RequestExpired = "RequestExpired"; + [[maybe_unused]] static const char* ServiceUnavailable = "ServiceUnavailable"; + [[maybe_unused]] static const char* ThrottlingException = "ThrottlingException"; + [[maybe_unused]] static const char* ValidationError = "ValidationError"; + [[maybe_unused]] static const char* DependencyTimeoutException = "DependencyTimeoutException"; + [[maybe_unused]] static const char* InvalidArnExceptio = "InvalidArnException"; + [[maybe_unused]] static const char* KMSInternalException = "KMSInternalException"; + [[maybe_unused]] static const char* NotFoundException = "NotFoundException"; + [[maybe_unused]] static const char* AlreadyExistsException = "AlreadyExistsException"; +} + +namespace beast = boost::beast; // from +// Note: switch http -> bhttp to deal with namespace ambiguity. +namespace bhttp = beast::http; // from +namespace shttp = seastar::http; + +static std::string to_lower(std::string_view s) { + std::string tmp(s.size(), 0); + std::transform(s.begin(), s.end(), tmp.begin(), ::tolower); + return tmp; +} + +static bool is_true(std::string_view s) { + auto tmp = to_lower(s); + return tmp == "true" || tmp == "1" || tmp == "yes" || tmp == "on"; +} + +class encryption::kms_host::impl { +public: + // set a rather long expiry. normal KMS policies are 365-day rotation of keys. + // we can do with 10 minutes. CMH. maybe even longer. + // (see comments below on what keys are here) + static inline constexpr std::chrono::milliseconds default_expiry = 600s; + static inline constexpr std::chrono::milliseconds default_refresh = 1200s; + + impl(encryption_context& ctxt, const std::string& name, const host_options& options) + : _ctxt(ctxt) + , _name(name) + , _options(options) + , _attr_cache(utils::loading_cache_config{ + .max_size = std::numeric_limits::max(), + .expiry = options.key_cache_expiry.value_or(default_expiry), + .refresh = options.key_cache_refresh.value_or(default_refresh)}, kms_log, std::bind(&impl::create_key, this, std::placeholders::_1)) + , _id_cache(utils::loading_cache_config{ + .max_size = std::numeric_limits::max(), + .expiry = options.key_cache_expiry.value_or(default_expiry), + .refresh = options.key_cache_refresh.value_or(default_refresh)}, kms_log, std::bind(&impl::find_key, this, std::placeholders::_1)) + { + // check if we have an explicit endpoint set. + if (!_options.endpoint.empty()) { + static std::regex simple_url(R"foo((https?):\/\/(?:([\w\.]+)|\[([\w:]+)\]):?(\d+)?\/?)foo"); + std::transform(_options.endpoint.begin(), _options.endpoint.end(), _options.endpoint.begin(), ::tolower); + std::smatch m; + if (!std::regex_match(_options.endpoint, m, simple_url)) { + throw std::invalid_argument(fmt::format("Could not parse URL: {}", _options.endpoint)); + } + _options.https = m[1].str() == "https"; + _options.host = m[2].length() > 0 ? m[2].str() : m[3].str(); + _options.port = m[4].length() > 0 ? std::stoi(m[4].str()) : 0; + } + if (_options.endpoint.empty() && _options.host.empty() && _options.aws_region.empty() && !_options.aws_use_ec2_region) { + throw std::invalid_argument("No AWS region or endpoint specified"); + } + if (_options.port == 0) { + _options.port = _options.https ? 443 : 80; + } + if (_options.aws_profile.empty()) { + auto profile = std::getenv("AWS_PROFILE"); + if (profile) { + _options.aws_profile = profile; + } else { + _options.aws_profile = "default"; + } + } + kms_log.trace("Added KMS node {}={}", name, _options.endpoint.empty() + ? (_options.host.empty() ? _options.aws_region : _options.host) + : _options.endpoint + ); + } + ~impl() = default; + + future<> init(); + const host_options& options() const { + return _options; + } + + future, id_type>> get_or_create_key(const key_info&, const option_override* = nullptr); + future> get_key_by_id(const id_type&, const key_info&, const option_override* = nullptr); +private: + class httpclient; + using key_and_id_type = std::tuple, id_type>; + + struct attr_cache_key { + std::string master_key; + std::string aws_assume_role_arn; + key_info info; + + bool operator==(const attr_cache_key& v) const = default; + friend std::ostream& operator<<(std::ostream& os, const attr_cache_key& k) { + fmt::print(os, "{}", std::tie(k.master_key, k.aws_assume_role_arn, k.info)); + return os; + } + }; + + struct attr_cache_key_hash { + size_t operator()(const attr_cache_key& k) const { + return utils::tuple_hash()(std::tie(k.master_key, k.aws_assume_role_arn, k.info.len)); + } + }; + + struct id_cache_key { + id_type id; + std::string aws_assume_role_arn; + bool operator==(const id_cache_key& v) const = default; + friend std::ostream& operator<<(std::ostream& os, const id_cache_key& k) { + fmt::print(os, "{{{}, {}}}", k.id, k.aws_assume_role_arn); + return os; + } + }; + + struct id_cache_key_hash { + size_t operator()(const id_cache_key& k) const { + return utils::tuple_hash()(std::tie(k.id, k.aws_assume_role_arn)); + } + }; + + struct aws_query; + using result_type = bhttp::response; + + future post(aws_query); + future post(std::string_view target, std::string_view aws_assume_role_arn, const rjson::value& query); + + future create_key(const attr_cache_key&); + future find_key(const id_cache_key&); + + encryption_context& _ctxt; + std::string _name; + host_options _options; + utils::loading_cache, attr_cache_key_hash> _attr_cache; + utils::loading_cache, id_cache_key_hash> _id_cache; + shared_ptr _creds; + std::unordered_map> _cache; + bool _initialized = false; +}; + +template <> struct fmt::formatter : fmt::ostream_formatter {}; +template <> struct fmt::formatter : fmt::ostream_formatter {}; + +/** + * Not in seastar. Because nowhere near complete, thought through or + * capable of dealing with anything but tiny aws messages. + * + * TODO: formalize and move to seastar + */ +class encryption::kms_host::impl::httpclient { +public: + httpclient(std::string host, uint16_t port, shared_ptr = {}); + + httpclient& add_header(std::string_view key, std::string_view value); + void clear_headers(); + + using result_type = kms_host::impl::result_type; + using request_type = bhttp::request; + + future send(); + + using method_type = bhttp::verb; + + void method(method_type); + void content(std::string_view); + void target(std::string_view); + + request_type& request() { + return _req; + } + const request_type& request() const { + return _req; + } + const std::string& host() const { + return _host; + } + uint16_t port() const { + return _port; + } +private: + + std::string _host; + uint16_t _port; + shared_ptr _creds; + request_type _req; +}; + +encryption::kms_host::impl::httpclient::httpclient(std::string host, uint16_t port, shared_ptr creds) + : _host(std::move(host)) + , _port(port) + , _creds(std::move(creds)) +{} + +encryption::kms_host::impl::httpclient& encryption::kms_host::impl::httpclient::add_header(std::string_view key, std::string_view value) { + _req.set(beast::string_view(key.data(), key.size()), beast::string_view(value.data(), value.size())); + return *this; +} + +void encryption::kms_host::impl::httpclient::clear_headers() { + _req.clear(); +} + +future encryption::kms_host::impl::httpclient::send() { + auto addr = co_await net::dns::resolve_name(_host); + socket_address sa(addr, _port); + connected_socket s = co_await (_creds + ? tls::connect(_creds, sa) + : seastar::connect(sa) + ); + + s.set_keepalive(true); + s.set_nodelay(true); + + auto out = s.output(); + auto in = s.input(); + + bhttp::serializer ser(_req); + + beast::error_code ec; + std::exception_ptr ex; + + bhttp::parser p(result_type{}); + + try { + while (!ser.is_done()) { + future<> f = make_ready_future<>(); + ser.next(ec, [&](beast::error_code& ec, auto&& buffers) { + for (auto const buffer : beast::buffers_range (buffers)) { + f = f.then([&out, data = buffer.data(), size = buffer.size()] { + return out.write(static_cast(data), size); + }); + } + ser.consume(beast::buffer_bytes(buffers)); + }); + + co_await std::move(f); + + if (ec.failed()) { + break; + } + } + + co_await out.flush(); + + p.eager(true); + p.skip(false); + + if (!ec.failed()) { + while (!p.is_done()) { + auto buf = co_await in.read(); + if (buf.empty()) { + break; + } + // parse + boost::asio::const_buffer wrap(buf.get(), buf.size()); + p.put(wrap, ec); + if (ec.failed() && ec != bhttp::error::need_more) { + break; + } + ec.clear(); + } + } + } catch (...) { + ex = std::current_exception(); + } + + try { + co_await out.close(); + } catch (...) { + if (!ex) { + ex = std::current_exception(); + } + } + try { + co_await in.close(); + } catch (...) { + if (!ex) { + ex = std::current_exception(); + } + } + + if (ec.failed()) { + throw std::system_error(ec); + } + if (ex) { + std::rethrow_exception(ex); + } + + co_return p.release(); +} + +void encryption::kms_host::impl::httpclient::method(method_type m) { + _req.method(m); +} + +void encryption::kms_host::impl::httpclient::content(std::string_view body) { + _req.body().assign(body.begin(), body.end()); + _req.set(bhttp::field::content_length, std::to_string(_req.body().size())); +} + +void encryption::kms_host::impl::httpclient::target(std::string_view target) { + _req.target(std::string(target)); +} + +static std::string get_option(const encryption::kms_host::option_override* oov, std::optional encryption::kms_host::option_override::* f, const std::string& def) { + if (oov) { + return (oov->*f).value_or(def); + } + return {}; +}; + +[[noreturn]] +static void translate_kms_error(const kms_error& e) { + using namespace kms_errors; + using namespace encryption; + + if (e.type() == AccessDeniedException || e.type() == MissingAuthenticationToken || e.type() == NotAuthorized) { + std::throw_with_nested(permission_error(e.what())); + } + if (e.type() == OptInRequired || e.type() == InvalidClientTokenId || e.type() == InvalidAction) { + std::throw_with_nested(configuration_error(e.what())); + } + if (e.type() == NotFoundException || e.type() == DependencyTimeoutException) { + std::throw_with_nested(missing_resource_error(e.what())); + } + std::throw_with_nested(service_error(e.what())); +} + +future, encryption::kms_host::id_type>> encryption::kms_host::impl::get_or_create_key(const key_info& info, const option_override* oov) { + attr_cache_key key { + .master_key = get_option(oov, &option_override::master_key, _options.master_key), + .aws_assume_role_arn = get_option(oov, &option_override::aws_assume_role_arn, _options.aws_assume_role_arn), + .info = info, + }; + + if (key.master_key.empty() && _options.master_key.empty()) { + throw configuration_error("No master key set in kms host config or encryption attributes"); + } + try { + co_return co_await _attr_cache.get(key); + } catch (kms_error& e) { + translate_kms_error(e); + } catch (base_error&) { + throw; + } catch (std::system_error& e) { + std::throw_with_nested(network_error(e.what())); + } catch (...) { + std::throw_with_nested(service_error(fmt::format("get_key_by_id: {}", std::current_exception()))); + } +} + +future> encryption::kms_host::impl::get_key_by_id(const id_type& id, const key_info& info, const option_override* oov) { + // note: since KMS does not really have any actual "key" associtation of id -> key, + // we only cache/query raw bytes of some length. (See below). + // Thus keys returned are always new objects. But they are not huge... + id_cache_key key { + .id = id, + .aws_assume_role_arn = get_option(oov, &option_override::aws_assume_role_arn, _options.aws_assume_role_arn), + }; + try { + auto data = co_await _id_cache.get(key); + co_return make_shared(info, data); + } catch (kms_error& e) { + translate_kms_error(e); + } catch (base_error&) { + throw; + } catch (std::system_error& e) { + std::throw_with_nested(network_error(e.what())); + } catch (std::invalid_argument& e) { + std::throw_with_nested(configuration_error(fmt::format("get_key_by_id: {}", e.what()))); + } catch (...) { + std::throw_with_nested(service_error(fmt::format("get_key_by_id: {}", std::current_exception()))); + } +} + +std::string make_aws_host(std::string_view aws_region, std::string_view service) { + static const char AWS_GLOBAL[] = "aws-global"; + static const char US_EAST_1[] = "us-east-1"; // US East (N. Virginia) + static const char CN_NORTH_1[] = "cn-north-1"; // China (Beijing) + static const char CN_NORTHWEST_1[] = "cn-northwest-1"; // China (Ningxia) + static const char US_ISO_EAST_1[] = "us-iso-east-1"; // US ISO East + static const char US_ISOB_EAST_1[] = "us-isob-east-1"; // US ISOB East (Ohio) + + // Fallback to us-east-1 if global endpoint does not exists. + auto region = aws_region == AWS_GLOBAL ? US_EAST_1 : aws_region; + + std::stringstream ss; + ss << service << "." << region; + + if (region == CN_NORTH_1 || region == CN_NORTHWEST_1) { + ss << ".amazonaws.com.cn"; + } else if (region == US_ISO_EAST_1) { + ss << ".c2s.ic.gov"; + } else if (region == US_ISOB_EAST_1) { + ss << ".sc2s.sgov.gov"; + } else { + ss << ".amazonaws.com"; + } + return ss.str(); +} + +struct encryption::kms_host::impl::aws_query { + std::string_view host; + + std::string_view service; + std::string_view target; + std::string_view content_type; + std::string_view content; + + std::string_view aws_access_key_id; + std::string_view aws_secret_access_key; + std::string_view security_token; + + uint16_t port; +}; + +future encryption::kms_host::impl::post(std::string_view target, std::string_view aws_assume_role_arn, const rjson::value& query) { + static auto get_response_error = [](const result_type& res) -> std::string { + switch (res.result()) { + case bhttp::status::unauthorized: case bhttp::status::forbidden: return "AccessDenied"; + case bhttp::status::not_found: return "ResourceNotFound"; + case bhttp::status::too_many_requests: return "SlowDown"; + case bhttp::status::internal_server_error: return "InternalError"; + case bhttp::status::service_unavailable: return "ServiceUnavailable"; + case bhttp::status::request_timeout: case bhttp::status::gateway_timeout: + case bhttp::status::network_connect_timeout_error: + return "RequestTimeout"; + default: + return format("{}", res.result()); + } + }; + + static auto query_ec2_meta = [](std::string_view target, std::string token = {}) -> future> { + static auto get_env_def = [](std::string_view var, std::string_view def) { + auto val = std::getenv(var.data()); + return val ? std::string_view(val) : def; + }; + + static const std::string ec2_meta_host(get_env_def("AWS_EC2_METADATA_ADDRESS", "169.254.169.254")); + static const int ec2_meta_port = std::stoi(get_env_def("AWS_EC2_METADATA_PORT", "80").data()); + + kms_log.debug("Query ec2 metadata"); + + httpclient client(ec2_meta_host, ec2_meta_port); + + static constexpr auto X_AWS_EC2_METADATA_TOKEN_TTL_SECONDS = "X-aws-ec2-metadata-token-ttl-seconds"; + static constexpr auto X_AWS_EC2_METADATA_TOKEN = "X-aws-ec2-metadata-token"; + static constexpr const char* HOST_HEADER = "host"; + + static auto logged_send = [](httpclient& client) -> future { + kms_log.trace("Request: {}", client.request()); + result_type res; + try { + res = co_await client.send(); + } catch (std::system_error& e) { + std::throw_with_nested(network_error(fmt::format("Error sending to host {}:{}: {}", client.host(), client.port(), e.what()))); + } catch (std::exception& e) { + std::throw_with_nested(service_error(fmt::format("Error sending to host {}:{}: {}", client.host(), client.port(), e.what()))); + } + kms_log.trace("Result: status={}, response={}", res.result_int(), res); + if (res.result() != bhttp::status::ok) { + throw kms_error(get_response_error(res), "EC2 metadata query"); + } + co_return res; + }; + + client.add_header(HOST_HEADER, ec2_meta_host); + + if (token.empty()) { + client.add_header(X_AWS_EC2_METADATA_TOKEN_TTL_SECONDS, "21600"); + client.method(httpclient::method_type::put); + client.target("/latest/api/token"); + + + auto res = co_await logged_send(client); + + if (res.result() != bhttp::status::ok) { + throw kms_error(get_response_error(res), "EC2 metadata token query"); + } + + token = res.body(); + client.clear_headers(); + } + + client.add_header(X_AWS_EC2_METADATA_TOKEN, token); + client.add_header(HOST_HEADER, ec2_meta_host); + client.method(httpclient::method_type::get); + client.target(target); + + auto res = co_await logged_send(client); + co_return std::make_tuple(std::move(res), token); + }; + + std::string gtoken; + + if (_options.aws_region.empty() && _options.host.empty()) { + assert(_options.aws_use_ec2_region); + httpclient::result_type res; + std::tie(res, gtoken) = co_await query_ec2_meta("/latest/meta-data/placement/region"); + _options.aws_region = res.body(); + } + + if (_options.host.empty()) { + // resolve region -> endpoint + assert(!_options.aws_region.empty()); + _options.host = make_aws_host(_options.aws_region, "kms"); + } + + auto should_resolve_options_credentials = [this] { + if (_options.aws_use_ec2_credentials) { + return false; + } + return _options.aws_access_key_id.empty() || _options.aws_secret_access_key.empty(); + }; + + // if we did not get full auth info in config, we can try to + // retrieve it from environment + if (should_resolve_options_credentials()) { + auto key_id = std::getenv("AWS_ACCESS_KEY_ID"); + auto key = std::getenv("AWS_SECRET_ACCESS_KEY"); + if (_options.aws_access_key_id.empty() && key_id) { + kms_log.debug("No aws id specified. Using environment AWS_ACCESS_KEY_ID"); + _options.aws_access_key_id = key_id; + } + if (_options.aws_secret_access_key.empty() && key) { + kms_log.debug("No aws secret specified. Using environment AWS_SECRET_ACCESS_KEY"); + _options.aws_secret_access_key = key; + } + } + + // if we did not get full auth info in config or env, we can try to + // retrieve it from ~/.aws/credentials + if (should_resolve_options_credentials()) { + auto home = std::getenv("HOME"); + if (home) { + auto credentials = std::string(home) + "/.aws/credentials"; + auto credentials_exists = co_await seastar::file_exists(credentials); + if (credentials_exists) { + kms_log.debug("No aws id/secret specified. Trying to read credentials from {}", credentials); + try { + auto buf = co_await read_text_file_fully(credentials); + std::string profile; + + static std::regex cred_line(R"foo(\s*\[(?:profile\s+)?(\w+)\]|([^\s]+)\s*=\s*([^\s]+)\s*\n)foo"); + std::cregex_iterator i(buf.get(), buf.get() + buf.size(), cred_line), e; + + std::string id, secret; + while (i != e) { + if ((*i)[1].length() > 0) { + profile = (*i)[1].str(); + kms_log.trace("Found profile {} ({})", profile, credentials); + } else if (profile == _options.aws_profile) { + std::string key((*i)[2].str()); + std::string val((*i)[3].str()); + if (key == "aws_access_key_id") { + id = val; + } else if (key == "aws_secret_access_key") { + secret = val; + } + } + ++i; + } + + if (!id.empty() && !_options.aws_access_key_id.empty() && id != _options.aws_access_key_id) { + throw configuration_error(fmt::format("Mismatched aws id: {} != {}", id, _options.aws_access_key_id)); + } + if (!id.empty() && _options.aws_access_key_id.empty()) { + _options.aws_access_key_id = id; + } + if (!secret.empty() && _options.aws_secret_access_key.empty()) { + _options.aws_secret_access_key = secret; + } + if (_options.aws_access_key_id.empty() || _options.aws_secret_access_key.empty()) { + throw configuration_error(fmt::format("Could not find credentials for profile {}", _options.aws_profile)); + } + kms_log.debug("Read credentials from {} ({}:{}{})", credentials, _options.aws_access_key_id + , _options.aws_secret_access_key.substr(0, 2) + , std::string(_options.aws_secret_access_key.size()-2, '-') + ); + } catch (...) { + kms_log.debug("Could not read credentials: {}", std::current_exception()); + } + } + } + } + + auto aws_access_key_id = _options.aws_access_key_id; + auto aws_secret_access_key = _options.aws_secret_access_key; + auto session = ""s; + + if (_options.aws_use_ec2_credentials) { + auto [res, token] = co_await query_ec2_meta("/latest/meta-data/iam/security-credentials/", gtoken); + auto role = res.body(); + + std::tie(res, std::ignore) = co_await query_ec2_meta("/latest/meta-data/iam/security-credentials/" + role, token); + auto body = rjson::parse(std::string_view(res.body().data(), res.body().size())); + + try { + aws_access_key_id = rjson::get(body, "AccessKeyId"); + aws_secret_access_key = rjson::get(body, "SecretAccessKey"); + session = rjson::get(body, "Token"); + } catch (rjson::malformed_value&) { + std::throw_with_nested(kms_error("AccessDenied", fmt::format("Code={}, Message={}" + , rjson::get_opt(body, "Code") + , rjson::get_opt(body, "Message") + ))); + } + } + + // Note: allowing user code to potentially reset aws_assume_role_arn='' -> no assumerole. + // Not 100% sure this is needed. + + if (!aws_assume_role_arn.empty()) { + auto sts_host = make_aws_host(_options.aws_region, "sts"); + auto now = db_clock::now(); + auto rs_id = utils::UUID_gen::get_time_UUID(std::chrono::system_clock::time_point(now.time_since_epoch())); + auto role_session = fmt::format("ScyllaDB-{}", rs_id); + + kms_log.debug("Assume role: {} (RoleSessionID={})", aws_assume_role_arn, role_session); + + auto res = co_await post(aws_query{ + .host = sts_host, + .service = "sts", + .content_type = "application/x-www-form-urlencoded; charset=utf-8", + .content = "Action=AssumeRole&Version=2011-06-15&RoleArn=" + + shttp::internal::url_encode(aws_assume_role_arn) + + "&RoleSessionName=" + role_session, + .aws_access_key_id = aws_access_key_id, + .aws_secret_access_key = aws_secret_access_key, + .security_token = session, + .port = _options.port, + }); + + if (res.result() != bhttp::status::ok) { + throw kms_error(get_response_error(res), "AssumeRole"); + } + + rapidxml::xml_document<> doc; + try { + doc.parse<0>(res.body().data()); + + using node_type = rapidxml::xml_node; + static auto get_xml_node = [](node_type* node, const char* what) { + auto res = node->first_node(what); + if (!res) { + throw kms_error("XML parse error", what); + } + return res; + }; + + auto arrsp = get_xml_node(&doc, "AssumeRoleResponse"); + auto arres = get_xml_node(arrsp, "AssumeRoleResult"); + auto creds = get_xml_node(arres, "Credentials"); + auto keyid = get_xml_node(creds, "AccessKeyId"); + auto key = get_xml_node(creds, "SecretAccessKey"); + auto token = get_xml_node(creds, "SessionToken"); + + aws_access_key_id = keyid->value(); + aws_secret_access_key = key->value(); + session = token->value(); + + } catch (const rapidxml::parse_error& e) { + std::throw_with_nested(kms_error("XML parse error", "AssumeRole")); + } + } + + auto res = co_await post(aws_query{ + .host = _options.host, + .service = "kms", + .target = target, + .content_type = "application/x-amz-json-1.1", + .content = rjson::print(query), + .aws_access_key_id = aws_access_key_id, + .aws_secret_access_key = aws_secret_access_key, + .security_token = session, + .port = _options.port, + }); + + auto body = rjson::empty_object(); + + if (!res.body().empty()) { + try { + body = rjson::parse(std::string_view(res.body().data(), res.body().size())); + } catch (...) { + if (res.result() == bhttp::status::ok) { + throw; + } + // assume non-json formatted error. fall back to parsing below + } + } + + if (res.result() != bhttp::status::ok) { + // try to format as good an error as we can. + static const char* message_lc_header = "message"; + static const char* message_cc_header = "Message"; + static const char* error_type_header = "x-amzn-ErrorType"; + static const char* type_header = "__type"; + + auto o = rjson::get_opt(body, message_lc_header); + if (!o) { + o = rjson::get_opt(body, message_cc_header); + } + auto msg = o.value_or("Unknown error"); + + o = rjson::get_opt(body, error_type_header); + if (!o) { + o = rjson::get_opt(body, type_header); + } + // this should never happen with aws, but... + auto type = o ? *o : get_response_error(res); + + throw kms_error(type, msg); + } + + co_return body; +} + +// helper to build AWS request and parse result. +future encryption::kms_host::impl::post(aws_query query) { + auto creds = _creds; + // if we are https, we need at least a credentials object that says "use system trust" + if (!creds && _options.https) { + creds = ::make_shared(); + + if (!_options.priority_string.empty()) { + creds->set_priority_string(_options.priority_string); + } else { + creds->set_priority_string(db::config::default_tls_priority); + } + + if (!_options.certfile.empty()) { + co_await creds->set_x509_key_file(_options.certfile, _options.keyfile, seastar::tls::x509_crt_format::PEM); + } + if (!_options.truststore.empty()) { + co_await creds->set_x509_trust_file(_options.truststore, seastar::tls::x509_crt_format::PEM); + } else { + co_await creds->set_system_trust(); + } + _creds = creds; + } + + // some of this could be shared with alternator + static constexpr const char* CONTENT_TYPE_HEADER = "content-type"; + static constexpr const char* HOST_HEADER = "host"; + static constexpr const char* X_AWS_DATE_HEADER = "X-Amz-Date"; + static constexpr const char* AWS_AUTHORIZATION_HEADER = "authorization"; + static constexpr const char* AMZ_SDK_INVOCATION_ID = "amz-sdk-invocation-id"; + static constexpr const char* X_AMZ_SECURITY_TOKEN = "X-Amz-Security-Token"; + + static constexpr const char* AMZ_TARGET_HEADER = "x-amz-target"; + static constexpr const char* AWS_HMAC_SHA256 = "AWS4-HMAC-SHA256"; + static constexpr const char* AWS4_REQUEST = "aws4_request"; + static constexpr const char* SIGNING_KEY = "AWS4"; + static constexpr const char* CREDENTIAL = "Credential"; + static constexpr const char* SIGNATURE = "Signature"; + static constexpr const char* SIGNED_HEADERS = "SignedHeaders"; + [[maybe_unused]] static constexpr const char* ACTION_HEADER = "Action"; + + static constexpr const char* ISO_8601_BASIC = "{:%Y%m%dT%H%M%SZ}"; + static constexpr const char* SIMPLE_DATE_FORMAT_STR = "{:%Y%m%d}"; + static constexpr auto NEWLINE = '\n'; + + auto now = db_clock::now(); + auto req_id = utils::UUID_gen::get_time_UUID(std::chrono::system_clock::time_point(now.time_since_epoch())); + + kms_log.trace("Building request: {} ({}:{}) {}", query.target, query.host, query.port, req_id); + + httpclient client(std::string(query.host), query.port, std::move(creds)); + + auto t_now = fmt::gmtime(db_clock::to_time_t(now)); + auto timestamp = fmt::format(ISO_8601_BASIC, t_now); + + // see https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html + // see AWS SDK. + // see https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html + std::stringstream signedHeadersStream; + std::stringstream canonicalRequestStream; + + canonicalRequestStream + << "POST" << NEWLINE + << "/" << NEWLINE << NEWLINE + ; + + auto add_signed_header = [&](std::string_view name, std::string_view value) { + client.add_header(name, value); + auto lname = to_lower(name); + canonicalRequestStream << lname << ":" << value << NEWLINE; + if (signedHeadersStream.tellp() != 0) { + signedHeadersStream << ';'; + } + signedHeadersStream << lname; + }; + + // headers must be sorted! + + add_signed_header(CONTENT_TYPE_HEADER, query.content_type); + add_signed_header(HOST_HEADER, query.host); + add_signed_header(X_AWS_DATE_HEADER, timestamp); + if (!query.target.empty()) { + add_signed_header(AMZ_TARGET_HEADER, "TrentService."s + std::string(query.target)); + } + + if (!query.security_token.empty()) { + //add_signed_header(X_AMZ_SECURITY_TOKEN, query.security_token); + client.add_header(X_AMZ_SECURITY_TOKEN, query.security_token); + } + + client.add_header(AMZ_SDK_INVOCATION_ID, fmt::format("{}", req_id)); + client.add_header("Accept-Encoding", "identity"); + client.add_header("Accept", "*/*"); + + auto make_hash = [&](std::string_view s) { + auto sha256 = calculate_sha256(bytes_view(reinterpret_cast(s.data()), s.size())); + auto hash = to_hex(sha256); + return hash; + }; + + auto hash = make_hash(query.content); + + auto signedHeadersValue = signedHeadersStream.str(); + canonicalRequestStream << NEWLINE << signedHeadersValue << NEWLINE << hash; + auto canonicalRequestString = canonicalRequestStream.str(); + auto canonicalRequestHash = make_hash(canonicalRequestString); + + kms_log.trace("Canonical request: {}", canonicalRequestString); + + auto simpleDate = fmt::format(SIMPLE_DATE_FORMAT_STR, t_now); + + std::stringstream stringToSignStream; + stringToSignStream << AWS_HMAC_SHA256 << NEWLINE + << timestamp << NEWLINE + << simpleDate << "/" << _options.aws_region << "/" + << query.service << "/" << AWS4_REQUEST << NEWLINE + << canonicalRequestHash + ; + auto stringToSign = stringToSignStream.str(); + + // these log messages intentionally made to mimic aws sdk/boto3 + kms_log.trace("StringToSign: {}", stringToSign); + + std::string finalSignature; + + { + auto tobv = [](std::string_view s) { + return bytes_view(reinterpret_cast(s.data()), s.size()); + }; + + auto signingKey = SIGNING_KEY + std::string(query.aws_secret_access_key); + auto kDate = hmac_sha256(tobv(simpleDate), tobv(signingKey)); + auto kRegion = hmac_sha256(tobv(_options.aws_region), kDate); + auto kService = hmac_sha256(tobv(query.service), kRegion); + auto hashResult = hmac_sha256(tobv(AWS4_REQUEST), kService); + auto finalHash = hmac_sha256(tobv(stringToSign), hashResult); + finalSignature = to_hex(finalHash); + } + + std::stringstream authStream; + authStream << AWS_HMAC_SHA256 << " " + << CREDENTIAL << "=" << query.aws_access_key_id << "/" << simpleDate << "/" << _options.aws_region + << "/" << query.service << "/" << AWS4_REQUEST << ", " << SIGNED_HEADERS + << "=" << signedHeadersValue << ", " << SIGNATURE << "=" << finalSignature + ; + + auto awsAuthString = authStream.str(); + + client.add_header(AWS_AUTHORIZATION_HEADER, awsAuthString); + client.target("/"); + client.content(query.content); + client.method(httpclient::method_type::post); + + kms_log.trace("Request: {}", client.request()); + + auto res = co_await client.send(); + + kms_log.trace("Result: status={}, response={}", res.result_int(), res); + + co_return res; +} + +static std::optional make_opt(const std::string& s) { + if (s.empty()) { + return std::nullopt; + } + return s; +} + +future<> encryption::kms_host::impl::init() { + if (_initialized) { + co_return; + } + + if (!_options.master_key.empty()) { + kms_log.debug("Looking up master key"); + auto query = rjson::empty_object(); + rjson::add(query, "KeyId", _options.master_key); + auto response = co_await post("DescribeKey", _options.aws_assume_role_arn, query); + kms_log.debug("Master key exists"); + } else { + kms_log.info("No default master key configured. Not verifying."); + } + _initialized = true; +} + +future encryption::kms_host::impl::create_key(const attr_cache_key& k) { + auto& master_key = k.master_key; + auto& aws_assume_role_arn = k.aws_assume_role_arn; + auto& info = k.info; + + /** + * AWS KMS does _not_ allow us to actually have "named keys" that can be used externally, + * i.e. exported to us, here, for bulk encryption. + * All named keys are 100% internal, the only options we have is using the + * "GenerateDataKey" API. This creates a new (epiphermal) key, encrypts it + * using a named (internal) key, and gives us both raw and encrypted blobs + * for usage as a local key. + * To be able to actually re-use this key again, on decryption of data, + * we employ the strategy recommended (https://docs.aws.amazon.com/kms/latest/APIReference/API_GenerateDataKey.html) + * namely actually embedding the encrypted key in the key ID associated with + * the locally encrypted data. So ID:s become pretty big. + * + * For ID -> key, we simply split the ID into the encrypted key part, and + * the master key name part, decrypt the first using the second (AWS KMS Decrypt), + * and create a local key using the result. + * + * Data recovery: + * Assuming you have data encrypted using a KMS generated key, you will have + * metadata detailing algorithm, key length etc (see sstable metadata, and key info). + * Metadata will also include a byte blob representing the ID of the encryption key. + * For KMS, the ID will actually be a text string: + * : + * + * I.e. something like: + * 761f258a-e2e9-40b3-8891-602b1b8b947e:e56sadfafa3324ff=/wfsdfwssdf + * or + * arn:aws:kms:us-east-1:797456418907:key/761f258a-e2e9-40b3-8891-602b1b8b947e:e56sadfafa3324ff=/wfsdfwssdf + * + * (last colon is separator) + * + * The actual data key can be retreived by doing a KMS "Decrypt" of the data blob part + * using the KMS key referenced by the key ID. This gives back actual key data that can + * be used to create a symmetric_key with algo, length etc as specified by metadata. + * + */ + + // avoid creating too many keys and too many calls. If we are not shard 0, delegate there. + if (this_shard_id() != 0) { + auto [data, id] = co_await smp::submit_to(0, [this, info, master_key, aws_assume_role_arn]() -> future> { + auto host = _ctxt.get_kms_host(_name); + option_override oov { + .master_key = make_opt(master_key), + .aws_assume_role_arn = make_opt(aws_assume_role_arn), + }; + auto [k, id] = co_await host->_impl->get_or_create_key(info, &oov); + co_return std::make_tuple(k != nullptr ? k->key() : bytes{}, id); + }); + co_return key_and_id_type{ + data.empty() ? nullptr : make_shared(info, data), + id + }; + } + + // note: since external keys are _not_ stored, + // there is nothing we can "look up" or anything. Always + // new key here. + + kms_log.debug("Creating new key: {}", info); + + auto query = rjson::empty_object(); + + rjson::add(query, "KeyId", std::string(master_key.begin(), master_key.end())); + rjson::add(query, "NumberOfBytes", info.len/8); + + auto response = co_await post("GenerateDataKey", aws_assume_role_arn, query); + auto data = base64_decode(rjson::get(response, "Plaintext")); + auto enc = rjson::get(response, "CiphertextBlob"); + auto kid = rjson::get(response, "KeyId"); + + try { + auto key = make_shared(info, data); + bytes id(kid.size() + 1 + enc.size(), 0); + auto i = std::copy(kid.begin(), kid.end(), id.begin()); + *i++ = ':'; + std::copy(enc.begin(), enc.end(), i); + + co_return key_and_id_type{ key, id }; + } catch (std::invalid_argument& e) { + std::throw_with_nested(configuration_error(e.what())); + } +} + +future encryption::kms_host::impl::find_key(const id_cache_key& k) { + // avoid creating too many keys and too many calls. If we are not shard 0, delegate there. + if (this_shard_id() != 0) { + co_return co_await smp::submit_to(0, [this, k]() -> future { + auto host = _ctxt.get_kms_host(_name); + auto bytes = co_await host->_impl->_id_cache.get(k); + co_return bytes; + }); + } + + // See create_key. ID consists of :. + // master id can (and will) contain ':', but blob will not. + // (we are being wasteful, and keeping the base64 encoding - easier to read) + auto& id = k.id; + auto pos = id.find_last_of(':'); + if (pos == id_type::npos) { + throw std::invalid_argument(format("Not a valid key id: {}", id)); + } + + kms_log.debug("Finding key: {}", id); + + std::string kid(id.begin(), id.begin() + pos); + std::string enc(id.begin() + pos + 1, id.end()); + + auto query = rjson::empty_object(); + rjson::add(query, "CiphertextBlob", enc); + rjson::add(query, "KeyId", kid); + + auto response = co_await post("Decrypt", k.aws_assume_role_arn, query); + auto data = base64_decode(rjson::get(response, "Plaintext")); + + // we know nothing about key type etc, so just return data. + co_return data; +} + +encryption::kms_host::kms_host(encryption_context& ctxt, const std::string& name, const host_options& options) + : _impl(std::make_unique(ctxt, name, options)) +{} + +encryption::kms_host::kms_host(encryption_context& ctxt, const std::string& name, const std::unordered_map& map) + : kms_host(ctxt, name, [&map] { + host_options opts; + map_wrapper> m(map); + + opts.aws_access_key_id = m("aws_access_key_id").value_or(""); + opts.aws_secret_access_key = m("aws_secret_access_key").value_or(""); + opts.aws_region = m("aws_region").value_or(""); + opts.aws_profile = m("aws_profile").value_or(""); + opts.aws_assume_role_arn = m("aws_assume_role_arn").value_or(""); + opts.aws_use_ec2_credentials = is_true(m("aws_use_ec2_credentials").value_or("false")); + opts.aws_use_ec2_region = is_true(m("aws_use_ec2_region").value_or("false")); + + // use "endpoint" semantics to match AWS configs. + opts.endpoint = m("endpoint").value_or(""); + opts.host = m("host").value_or(""); + opts.port = std::stoi(m("port").value_or("0")); + + opts.master_key = m("master_key").value_or(""); + opts.certfile = m("certfile").value_or(""); + opts.keyfile = m("keyfile").value_or(""); + opts.truststore = m("truststore").value_or(""); + opts.priority_string = m("priority_string").value_or(""); + + opts.key_cache_expiry = parse_expiry(m("key_cache_expiry")); + opts.key_cache_refresh = parse_expiry(m("key_cache_refresh")); + return opts; + }()) +{} + +encryption::kms_host::~kms_host() = default; + +future<> encryption::kms_host::init() { + return _impl->init(); +} + +const encryption::kms_host::host_options& encryption::kms_host::options() const { + return _impl->options(); +} + +future, encryption::kms_host::id_type>> encryption::kms_host::get_or_create_key(const key_info& info, const option_override* oov) { + return _impl->get_or_create_key(info, oov); +} + +future> encryption::kms_host::get_key_by_id(const id_type& id, const key_info& info, const option_override* oov) { + return _impl->get_key_by_id(id, info, oov); +} + diff --git a/ent/encryption/kms_host.hh b/ent/encryption/kms_host.hh new file mode 100644 index 0000000000..ac9fd1de41 --- /dev/null +++ b/ent/encryption/kms_host.hh @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2022 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +#include "symmetric_key.hh" + +namespace encryption { + +class encryption_context; +struct key_info; + +class kms_host { +public: + struct host_options { + std::string endpoint; + // or... + std::string host; + uint16_t port; + bool https = true; + // auth + std::string aws_access_key_id; + std::string aws_secret_access_key; + std::string aws_region; + std::string aws_profile; + std::string aws_assume_role_arn; + + bool aws_use_ec2_credentials; + bool aws_use_ec2_region; + + // key to use for keys + std::string master_key; + // tls. if unspeced, use system for https + // AWS does not (afaik?) allow certificate auth + // but we keep the option available just in case. + std::string certfile; + std::string keyfile; + std::string truststore; + std::string priority_string; + + std::optional key_cache_expiry; + std::optional key_cache_refresh; + }; + using id_type = bytes; + + kms_host(encryption_context&, const std::string& name, const host_options&); + kms_host(encryption_context&, const std::string& name, const std::unordered_map&); + ~kms_host(); + + future<> init(); + const host_options& options() const; + + struct option_override { + std::optional master_key; + std::optional aws_assume_role_arn; + }; + + future, id_type>> get_or_create_key(const key_info&, const option_override* = nullptr); + future> get_key_by_id(const id_type&, const key_info&, const option_override* = nullptr); +private: + class impl; + std::unique_ptr _impl; +}; + +} diff --git a/ent/encryption/kms_key_provider.cc b/ent/encryption/kms_key_provider.cc new file mode 100644 index 0000000000..3140de1513 --- /dev/null +++ b/ent/encryption/kms_key_provider.cc @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2022 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#include +#include + +#include "kms_key_provider.hh" +#include "kms_host.hh" + +namespace encryption { + +class kms_key_provider : public key_provider { +public: + kms_key_provider(::shared_ptr kms_host, std::string name, kms_host::option_override oov) + : _kms_host(std::move(kms_host)) + , _name(std::move(name)) + , _oov(std::move(oov)) + {} + future> key(const key_info& info, opt_bytes id) override { + if (id) { + return _kms_host->get_key_by_id(*id, info, &_oov).then([id](key_ptr k) { + return make_ready_future>(std::tuple(k, id)); + }); + } + return _kms_host->get_or_create_key(info, &_oov).then([](std::tuple k_id) { + return make_ready_future>(k_id); + }); + } + void print(std::ostream& os) const override { + os << _name; + } +private: + ::shared_ptr _kms_host; + std::string _name; + kms_host::option_override _oov; +}; + +shared_ptr kms_key_provider_factory::get_provider(encryption_context& ctxt, const options& map) { + opt_wrapper opts(map); + auto kms_host = opts("kms_host"); + kms_host::option_override oov { + .master_key = opts("master_key"), + .aws_assume_role_arn = opts("aws_assume_role_arn"), + }; + + if (!kms_host) { + throw std::invalid_argument("kms_host must be provided"); + } + + auto host = ctxt.get_kms_host(*kms_host); + auto id = kms_host.value() + + ":" + oov.master_key.value_or(host->options().master_key) + + ":" + oov.aws_assume_role_arn.value_or(host->options().aws_assume_role_arn) + ; + auto provider = ctxt.get_cached_provider(id); + + if (!provider) { + provider = ::make_shared(host, *kms_host, std::move(oov)); + ctxt.cache_provider(id, provider); + } + + return provider; +} + +} diff --git a/ent/encryption/kms_key_provider.hh b/ent/encryption/kms_key_provider.hh new file mode 100644 index 0000000000..ae0cf0eeba --- /dev/null +++ b/ent/encryption/kms_key_provider.hh @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2022 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include "encryption.hh" +#include "system_key.hh" + +namespace encryption { + +class kms_key_provider_factory : public key_provider_factory { +public: + shared_ptr get_provider(encryption_context&, const options&) override; +}; + +/** + * As it stands today, given system_key api (gives keys), and + * what it is used for (config encryption), we cannot provide + * a KMS system key. This is because: + * + * a.) KMS does not allow us to store a named object (key) in a secure(ish) way. + * We can encrypt/decrypt and create one-off keys for local usage, which are + * encoded in their own ID (see kms_host), but having a unique key from + * a "path" is not possible. Esp. due to key rotation, encrypted data preamble + * etc. We could keep the encrypted key material in a local file, then decrypt + * it using a named key on startup, but given b.) it is dubious if this is useful. + * b.) System keys are only used for config encryption. The authentication config for + * AWS/KMS access is typically one of the things that should be encrypted. Thus + * we would create a big chicken and egg problem here. + */ +} diff --git a/ent/encryption/local_file_provider.cc b/ent/encryption/local_file_provider.cc new file mode 100644 index 0000000000..2fe1a3e9fa --- /dev/null +++ b/ent/encryption/local_file_provider.cc @@ -0,0 +1,292 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include + +#include "local_file_provider.hh" +#include "symmetric_key.hh" +#include "encryption.hh" +#include "encryption_exceptions.hh" +#include "encryption_config.hh" +#include "db/config.hh" + +namespace encryption { + +namespace bfs = std::filesystem; + +const sstring default_key_file_path = (bfs::path(db::config::get_conf_dir()) / "data_encryption_keys").string(); + +static const key_info system_key_info{ "System", 0 }; + +class local_file_provider : public key_provider { +public: + local_file_provider(encryption_context& ctxt, const bfs::path& path, bool must_exist = false) + : local_file_provider(ctxt, sstring(bfs::absolute(path).string()), must_exist) + {} + local_file_provider(encryption_context& ctxt, const sstring& path, bool must_exist = false) + : _ctxt(ctxt) + , _path(path) + , _sem(1) + , _must_exist(must_exist) + {} + future> key(const key_info& info, opt_bytes = {}) override { + // TODO: assert options -> my key + auto i = _keys.find(info); + if (i != _keys.end()) { + return make_ready_future>(std::tuple(i->second, std::nullopt)); + } + return load_or_create(info).then([](key_ptr k) { + return make_ready_future>(std::tuple(k, std::nullopt)); + }); + } + future<> validate() const override { + auto f = make_ready_future<>(); + if (!_must_exist) { + return f; + } + // if we must exist, we don't change. Ok to open from all shards. + return f.then([this] { + return open_file_dma(_path, open_flags::ro).then([](file f) { + return f.close(); + }); + }).handle_exception([this](auto ep) { + try { + std::rethrow_exception(ep); + } catch (...) { + std::throw_with_nested(missing_resource_error("Could not read '" + _path + "'")); + } + }); + } + + const sstring& path() const { + return _path; + } + void print(std::ostream& os) const override { + os << "key=" << _path; + } + +private: + future load_or_create(const key_info&); + future load_or_create_local(const key_info&); + future<> read_key_file(); + future write_key_file(key_info); + + std::unordered_map _keys; + encryption_context& _ctxt; + sstring _path; + semaphore _sem; + bool _read_file = false; + bool _must_exist = false; +}; + +shared_ptr local_file_provider_factory::find(encryption_context& ctxt, const sstring& path) { + auto p = ctxt.get_cached_provider(path); + if (!p) { + p = make_shared(ctxt, path); + ctxt.cache_provider(path, p); + } + return p; +} + +shared_ptr local_file_provider_factory::get_provider(encryption_context& ctxt, const options& map) { + opt_wrapper opts(map); + return find(ctxt, opts(SECRET_KEY_FILE).value_or(default_key_file_path)); +} + +future +local_file_provider::load_or_create(const key_info& info) { + // if someone uses a system key as a table key, we could still race + // here. but that is a user error, so ignore + if (this_shard_id() == 0 || &info == &system_key_info) { + return load_or_create_local(info); + } + + struct data { + bytes key; + key_info info; + }; + + /** + * Key files are singular. Not sharded. This would be ok if we only read from them. + * But in keeping with dse compat, we don't. So rather than dealing with lock files + * or whatnot, we simply say that a single file is handled by a single key object, + * and only on shard 0. So if we are not shard 0, we call to there, find our + * counterpart object (local_file_provider_factory::find), and as him about the + * key data instead. He in turn will sync on his semaphore. + * + * The downside is that we are not resilient against multiple processes messing + * with the key file, but neither is dse + */ + return do_with(data{bytes(bytes::initialized_later(), info.len/8), info}, [this](data& i) { + return smp::submit_to(0, [this, &i]{ + auto kp = static_pointer_cast(local_file_provider_factory::find(_ctxt, _path)); + auto f = kp->load_or_create_local(i.info); + return f.then([&i, kp](key_ptr k) { + auto& kd = k->key(); + i.key.resize(kd.size()); + std::copy(kd.begin(), kd.end(), i.key.begin()); + }); + }).then([this, &i] { + auto k = make_shared(i.info, i.key); + _keys.emplace(i.info, k); + return make_ready_future(std::move(k)); + }); + }); +} + +future +local_file_provider::load_or_create_local(const key_info& info) { + if (_keys.count(info)) { + return make_ready_future(_keys.at(info)); + } + return read_key_file().then([this, info] { + if (_keys.count(info)) { + return make_ready_future(_keys.at(info)); + } + if (info == system_key_info) { + if (_keys.size() != 1) { + _keys.clear(); + return make_exception_future(std::invalid_argument("System key must contain exactly one entry")); + } + auto k = _keys.begin()->second; + _keys.clear(); + _keys.emplace(info, k); + return make_ready_future(k); + } + // create it. + return write_key_file(info); + }); +} + +future<> local_file_provider::read_key_file() { + if (_read_file) { + return make_ready_future(); + } + + // #1923 - a key can have a descriptor string line "AES:128:" iff user relies on + // defaults. Must match this as well. + static const std::regex key_line_expr(R"foo((\w+(?:\/\w+)?(?:\/\w+)?)\:(\d+)\:(\S+)\s*)foo"); + + return with_semaphore(_sem, 1, [this] { + // could do this twice, but it is only reading + return read_text_file_fully(_path).then([this](temporary_buffer buf) { + auto i = std::cregex_iterator(buf.begin(), buf.end(), key_line_expr); + auto e = std::cregex_iterator(); + + while (i != e) { + std::cmatch m = *i; + auto alg = m[1].str(); + auto len = std::stoul(m[2].str()); + auto key = m[3].str(); + + auto info = key_info{alg, unsigned(len)}; + if (!_keys.count(info)) { + auto kb = base64_decode(key); + auto k = make_shared(info, kb); + _keys.emplace(info, std::move(k)); + } + ++i; + } + _read_file = true; + }).handle_exception([this](auto ep) { + try { + std::rethrow_exception(ep); + } catch (std::system_error& e) { + if (e.code() == std::error_code(ENOENT, std::system_category())) { + if (!_must_exist) { + return; + } + std::throw_with_nested(configuration_error("Key file '" + _path + "' does not exist")); + } + std::throw_with_nested(service_error("read_key_file")); + } catch (std::invalid_argument& e) { + std::throw_with_nested(configuration_error(fmt::format("read_key_file: {}", e.what()))); + } catch (...) { + std::throw_with_nested(service_error(fmt::format("read_key_file: {}", std::current_exception()))); + } + }); + }); +} + +future local_file_provider::write_key_file(key_info info) { + return with_semaphore(_sem, 1, [this, info] { + // we can get here more than once if shards race. + // however, we only need to use/write the first key matching + // the required info. + if (_keys.count(info)) { + return make_ready_future(_keys.at(info)); + } + + auto k = make_shared(info); + + std::ostringstream ss; + for (auto& p : _keys) { + ss << p.first.alg << ":" << p.first.len << ":" << base64_encode(p.second->key()) << std::endl; + } + ss << info.alg << ":" << info.len << ":" << base64_encode(k->key()) << std::endl; + auto s = ss.str(); + auto tmpnam = _path + ".tmp"; + auto f = make_ready_future<>(); + if (!_must_exist) { + f = seastar::recursive_touch_directory((bfs::path(tmpnam).remove_filename()).string()); + } + return f.then([this, tmpnam, s] { + return write_text_file_fully(tmpnam, s).then([this, tmpnam] { + return rename_file(tmpnam, _path); + }); + }).then([this, k, info] { + // don't cache until written + _keys[info] = k; + return make_ready_future(k); + }); + }).handle_exception([this](auto ep) -> key_ptr{ + try { + std::rethrow_exception(ep); + } catch (...) { + std::throw_with_nested(service_error("Could not write key file '" + _path + "'")); + } + }); +} + +local_system_key::local_system_key(encryption_context& ctxt, const sstring& path) + : _provider(make_shared(ctxt, bfs::path(ctxt.config().system_key_directory()) / bfs::path(path), true)) +{} + +local_system_key::~local_system_key() +{} + +future> local_system_key::get_key() { + return _provider->key(system_key_info).then([](std::tuple k_id) { + return make_ready_future>(std::get<0>(std::move(k_id))); + }); +} + +future<> local_system_key::validate() const { + // first, just validate the file provider itself + co_await _provider->validate(); + // second, do an early load of the actual key to ensure file contents. + co_await _provider->key(system_key_info); +} + +const sstring& local_system_key::name() const { + return _provider->path(); +} + +} diff --git a/ent/encryption/local_file_provider.hh b/ent/encryption/local_file_provider.hh new file mode 100644 index 0000000000..9378e413c8 --- /dev/null +++ b/ent/encryption/local_file_provider.hh @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include "encryption.hh" +#include "system_key.hh" + +namespace encryption { + +const extern sstring default_key_file_path; + +class local_file_provider; + +class local_file_provider_factory : public key_provider_factory { +public: + static shared_ptr find(encryption_context&, const sstring& path); + shared_ptr get_provider(encryption_context&, const options&) override; +}; + +class local_system_key : public system_key { + shared_ptr _provider; +public: + local_system_key(encryption_context&, const sstring&); + ~local_system_key(); + + future> get_key() override; + future<> validate() const override; + const sstring& name() const override; + bool is_local() const override { + return true; + } +}; + +} diff --git a/ent/encryption/replicated_key_provider.cc b/ent/encryption/replicated_key_provider.cc new file mode 100644 index 0000000000..9a06950fcc --- /dev/null +++ b/ent/encryption/replicated_key_provider.cc @@ -0,0 +1,477 @@ +/* + * Copyright (C) 2015 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include "utils/to_string.hh" + +#include "replicated_key_provider.hh" +#include "encryption.hh" +#include "encryption_exceptions.hh" +#include "local_file_provider.hh" +#include "symmetric_key.hh" +#include "replica/database.hh" +#include "cql3/query_processor.hh" +#include "cql3/untyped_result_set.hh" +#include "utils/UUID.hh" +#include "utils/UUID_gen.hh" +#include "utils/hash.hh" +#include "service/storage_service.hh" +#include "service/migration_manager.hh" +#include "compaction/compaction_manager.hh" +#include "replica/distributed_loader.hh" +#include "schema/schema_builder.hh" +#include "db/system_keyspace.hh" +#include "db/extensions.hh" +#include "locator/everywhere_replication_strategy.hh" + +namespace encryption { + +static auto constexpr KSNAME = "system_replicated_keys"; +static auto constexpr TABLENAME = "encrypted_keys"; + +static logger log("replicated_key_provider"); + +using utils::UUID; + +class replicated_key_provider : public key_provider { +public: + static constexpr int8_t version = 0; + /** + * Header: + * 1 byte version + * 16 bytes UUID of key + * 16 bytes MD5 of UUID + */ + static const size_t header_size = 33; + + struct key_id { + key_info info; + opt_bytes id; + + key_id(key_info k, opt_bytes b = {}) + : info(std::move(k)) + , id(std::move(b)) + {} + bool operator==(const key_id& v) const { + return info == v.info && id == v.id; + } + }; + + struct key_id_hash { + size_t operator()(const key_id& id) const { + return utils::tuple_hash()(std::tie(id.info.alg, id.info.len, id.id)); + } + }; + + replicated_key_provider(encryption_context& ctxt, shared_ptr system_key, shared_ptr local_provider) + : _ctxt(ctxt) + , _system_key(std::move(system_key)) + , _local_provider(std::move(local_provider)) + {} + + + future> key(const key_info&, opt_bytes = {}) override; + future<> validate() const override; + future<> maybe_initialize_tables(); + static future<> do_initialize_tables(::replica::database& db, service::migration_manager&); + + bool should_delay_read(const opt_bytes& id) const override { + if (!id || _initialized) { + return false; + } + if (!_initialized) { + return true; + } + auto& qp = _ctxt.get_query_processor(); + // This check should be ok, and even somewhat redundant. "Initialized" above + // will only be set once we've generated/queried a key not passing through here + // (i.e. a key for write _or_ commit log (should we allow this)). This can only be + // done if: + // a.) Encryption was already set up, thus table existed and we waited + // for distributed_tables in "ensure_populated" + // b.) Encryption was added. In which case we are way past bootstrap + // and can receive user commands. + // c.) System table/commit log write, with either first use of this provider, + // in which case we're creating the table (here at least) - thus fine, + // or again, we've waited through "ensure_populated", so keys are + // readble. At worst, we create a few extra keys. + // Note: currently c.) is not relevant, as we don't support system/commitlog + // encryption using repl_prov. + return !qp.local_is_initialized(); + } + + void print(std::ostream& os) const override { + os << "system_key=" << _system_key->name() << ", local=" << *_local_provider; + } + +private: + void store_key(const key_id&, const UUID&, key_ptr); + + static opt_bytes decode_id(const opt_bytes&); + static bytes encode_id(const UUID&); + + future> get_key(const key_info&, opt_bytes = {}); + + future load_or_create(const key_info&); + future load_or_create_local(const key_info&); + future<> read_key_file(); + future<> write_key_file(); + + template + future<::shared_ptr> query(sstring, Args&& ...); + + future<> force_blocking_flush(); + + encryption_context& _ctxt; + shared_ptr _system_key; + shared_ptr _local_provider; + std::unordered_map, key_id_hash> _keys; + + bool _initialized = false; + bool _use_cache = true; + + friend class replicated_key_provider_factory; + + static const utils::UUID local_fallback_uuid; + static const bytes local_fallback_id; + static const bytes_view local_fallback_bytes; +}; + +using namespace std::chrono_literals; + +static const timeout_config rkp_db_timeout_config { + 5s, 5s, 5s, 5s, 5s, 5s, 5s, +}; + +static service::query_state& rkp_db_query_state() { + static thread_local service::client_state cs(service::client_state::internal_tag{}, rkp_db_timeout_config); + static thread_local service::query_state qs(cs, empty_service_permit()); + return qs; +} + +template +future<::shared_ptr> replicated_key_provider::query(sstring q, Args&& ...params) { + auto mode = co_await _ctxt.get_storage_service().local().get_operation_mode(); + if (mode != service::storage_service::mode::STARTING) { + co_return co_await _ctxt.get_query_processor().local().execute_internal(q, { std::forward(params)...}, cql3::query_processor::cache_internal::no); + } + co_return co_await _ctxt.get_query_processor().local().execute_internal(q, db::consistency_level::ONE, rkp_db_query_state(), { std::forward(params)...}, cql3::query_processor::cache_internal::no); +} + +future<> replicated_key_provider::force_blocking_flush() { + return _ctxt.get_database().invoke_on_all([](replica::database& db) { + // if (!Boolean.getBoolean("cassandra.unsafesystem")) + replica::column_family& cf = db.find_column_family(KSNAME, TABLENAME); + return cf.flush(); + }); +} + +void replicated_key_provider::store_key(const key_id& id, const UUID& uuid, key_ptr k) { + if (!_use_cache) { + return; + } + _keys[id] = std::make_pair(uuid, k); + if (!id.id) { + _keys[key_id(id.info, uuid.serialize())] = std::make_pair(uuid, k); + } +} + +opt_bytes replicated_key_provider::decode_id(const opt_bytes& b) { + if (b) { + auto i = b->begin(); + auto v = *i++; + if (v == version && b->size() == 33) { + bytes id(i + 1, i + 1 + 16); + bytes md(i + 1 + 16, b->end()); + if (calculate_md5(id) == md) { + return id; + } + } + } + return std::nullopt; +} + +bytes replicated_key_provider::encode_id(const UUID& uuid) { + bytes b{bytes::initialized_later(), header_size}; + auto i = b.begin(); + *i++ = version; + uuid.serialize(i); + auto md = calculate_md5(b, 1, 16); + std::copy(md.begin(), md.end(), i); + return b; +} + +const utils::UUID replicated_key_provider::local_fallback_uuid(0u, 0u); // not valid! +const bytes replicated_key_provider::local_fallback_id = encode_id(local_fallback_uuid); +const bytes_view replicated_key_provider::local_fallback_bytes(local_fallback_id.data() + 1, 16); + +future> replicated_key_provider::key(const key_info& info, opt_bytes input) { + opt_bytes id; + + if (input) { //reading header? + auto v = *input; + if (v[0] == version) { + bytes bid(v.begin() + 1, v.begin() + 1 + 16); + bytes md(v.begin() + 1 + 16, v.begin() + 1 + 32); + if (calculate_md5(bid) == md) { + id = bid; + } + } + } + + bool try_local = id == local_fallback_bytes; + + // if the id indicates the key came from local fallback, don't even + // try keyspace lookup. + if (!try_local) { + try { + auto [uuid, k] = co_await get_key(info, std::move(id)); + co_return std::make_tuple(k, encode_id(uuid)); + } catch (std::invalid_argument& e) { + std::throw_with_nested(configuration_error(e.what())); + } catch (...) { + auto ep = std::current_exception(); + log.warn("Exception looking up key {}: {}", info, ep); + if (_local_provider) { + try { + std::rethrow_exception(ep); + } catch (replica::no_such_keyspace&) { + } catch (exceptions::invalid_request_exception&) { + } catch (exceptions::read_failure_exception&) { + } catch (...) { + std::throw_with_nested(service_error(fmt::format("key: {}", std::current_exception()))); + } + if (!id) { + try_local = true; + } + } + if (!try_local) { + std::throw_with_nested(service_error(fmt::format("key: {}", std::current_exception()))); + } + } + } + + log.warn("Falling back to local key {}", info); + auto [k, nid] = co_await _local_provider->key(info, id); + if (nid && nid != id) { + // local provider does not give ids. + throw malformed_response_error("Expected null id back from local provider"); + } + co_return std::make_tuple(k, local_fallback_id); +} + +future> replicated_key_provider::get_key(const key_info& info, opt_bytes opt_id) { + if (!_initialized) { + co_await maybe_initialize_tables(); + } + + key_id id(info, std::move(opt_id)); + auto i = _keys.find(id); + if (i != _keys.end()) { + co_return std::tuple(i->second.first, i->second.second); + } + + // TODO: origin does non-cql acquire of all available keys from + // replicas in the "host_ids" table iff we get here during boot. + // For now, ignore this and assume that if we have a sstable with + // key X, we should have a local replica of X as well, given + // the "everywhere strategy of the keys table. + + auto cipher = info.alg.substr(0, info.alg.find('/')); // e.g. "AES" + + UUID uuid; + shared_ptr res; + + if (id.id) { + uuid = utils::UUID_gen::get_UUID(*id.id); + log.debug("Finding key {} ({})", uuid, info); + auto s = fmt::format("SELECT * FROM {}.{} WHERE key_file=? AND cipher=? AND strength=? AND key_id=?;", KSNAME, TABLENAME); + res = co_await query(std::move(s), _system_key->name(), cipher, int32_t(id.info.len), uuid); + + // if we find nothing, and we actually queried a specific key (by uuid), we've failed. + if (res->empty()) { + log.debug("Could not find key {}", id.id); + throw std::runtime_error(fmt::format("Unable to find key for cipher={} strength={} id={}", cipher, id.info.len, uuid)); + } + } else { + log.debug("Finding key ({})", info); + auto s = fmt::format("SELECT * FROM {}.{} WHERE key_file=? AND cipher=? AND strength=? LIMIT 1;", KSNAME, TABLENAME); + res = co_await query(std::move(s), _system_key->name(), cipher, int32_t(id.info.len)); + } + + // otoh, if we don't need a specific key, we can just create a new one (writing a sstable) + if (res->empty()) { + uuid = utils::UUID_gen::get_time_UUID(); + + log.debug("No key found. Generating {}", uuid); + + auto k = make_shared(id.info); + store_key(id, uuid, k); + + auto b = co_await _system_key->encrypt(k->key()); + auto ks = base64_encode(b); + log.trace("Inserting generated key {}", uuid); + co_await query(fmt::format("INSERT INTO {}.{} (key_file, cipher, strength, key_id, key) VALUES (?, ?, ?, ?, ?)", + KSNAME, TABLENAME), _system_key->name(), cipher, int32_t(id.info.len), uuid, ks + ); + log.trace("Flushing key table"); + co_await force_blocking_flush(); + + co_return std::tuple(uuid, k); + } + + // found it + auto& row = res->one(); + uuid = row.get_as("key_id"); + auto ks = row.get_as("key"); + auto kb = base64_decode(ks); + auto b = co_await _system_key->decrypt(kb); + auto k = make_shared(id.info, b); + store_key(id, uuid, k); + + co_return std::tuple(uuid, k); +} + +future<> replicated_key_provider::validate() const { + try { + co_await _system_key->validate(); + } catch (...) { + std::throw_with_nested(std::invalid_argument(fmt::format("Could not validate system key: {}", _system_key->name()))); + } + if (_local_provider){ + co_await _local_provider->validate(); + } +} + +schema_ptr encrypted_keys_table() { + static thread_local auto schema = [] { + auto id = generate_legacy_id(KSNAME, TABLENAME); + return schema_builder(KSNAME, TABLENAME, std::make_optional(id)) + .with_column("key_file", utf8_type, column_kind::partition_key) + .with_column("cipher", utf8_type, column_kind::partition_key) + .with_column("strength", int32_type, column_kind::clustering_key) + .with_column("key_id", timeuuid_type, column_kind::clustering_key) + .with_column("key", utf8_type) + .with_hash_version() + .build(); + }(); + return schema; +} + +future<> replicated_key_provider::maybe_initialize_tables() { + if (!_initialized) { + co_await do_initialize_tables(_ctxt.get_database().local(), _ctxt.get_migration_manager().local()); + _initialized = true; + } +} + +future<> replicated_key_provider::do_initialize_tables(::replica::database& db, service::migration_manager& mm) { + if (db.has_schema(KSNAME, TABLENAME)) { + co_return; + } + + log.debug("Creating keyspace and table"); + if (!db.has_keyspace(KSNAME)) { + auto group0_guard = co_await mm.start_group0_operation(); + auto ts = group0_guard.write_timestamp(); + try { + auto ksm = keyspace_metadata::new_keyspace( + KSNAME, + "org.apache.cassandra.locator.EverywhereStrategy", + {}, + std::nullopt, + true); + co_await mm.announce(service::prepare_new_keyspace_announcement(db, ksm, ts), std::move(group0_guard), fmt::format("encryption at rest: create keyspace {}", KSNAME)); + } catch (exceptions::already_exists_exception&) { + } + } + auto group0_guard = co_await mm.start_group0_operation(); + auto ts = group0_guard.write_timestamp(); + try { + co_await mm.announce(co_await service::prepare_new_column_family_announcement(mm.get_storage_proxy(), encrypted_keys_table(), ts), std::move(group0_guard), + fmt::format("encryption at rest: create table {}.{}", KSNAME, TABLENAME)); + } catch (exceptions::already_exists_exception&) { + } + auto& ks = db.find_keyspace(KSNAME); + auto& rs = ks.get_replication_strategy(); + // should perhaps check name also.. + if (rs.get_type() != locator::replication_strategy_type::everywhere_topology) { + // TODO: reset to everywhere + repair. + } +} + +const size_t replicated_key_provider::header_size; + +replicated_key_provider_factory::replicated_key_provider_factory() +{} + +replicated_key_provider_factory::~replicated_key_provider_factory() +{} + +namespace bfs = std::filesystem; + +shared_ptr replicated_key_provider_factory::get_provider(encryption_context& ctxt, const options& map) { + opt_wrapper opts(map); + auto system_key_name = opts(SYSTEM_KEY_FILE).value_or("system_key"); + if (system_key_name.find('/') != sstring::npos) { + throw std::invalid_argument("system_key cannot contain '/'"); + } + + auto system_key = ctxt.get_system_key(system_key_name); + auto local_key_file = bfs::absolute(bfs::path(opts(SECRET_KEY_FILE).value_or(default_key_file_path))); + + if (system_key->is_local() && bfs::absolute(bfs::path(system_key->name())) == local_key_file) { + throw std::invalid_argument("system key and local key cannot be the same"); + } + + auto name = system_key->name() + ":" + local_key_file.string(); + auto debug = opts("DEBUG"); + if (debug) { + name = name + ":" + *debug; + } + auto p = ctxt.get_cached_provider(name); + if (!p) { + auto rp = seastar::make_shared(ctxt, std::move(system_key), local_file_provider_factory::find(ctxt, local_key_file.string())); + ctxt.cache_provider(name, rp); + + if (debug && debug->find("nocache") != sstring::npos) { + log.debug("Turn off cache"); + rp->_use_cache = false; + } + p = std::move(rp); + } + + return p; +} + +void replicated_key_provider_factory::init(db::extensions& exts) { + exts.add_extension_internal_keyspace(KSNAME); +} + +future<> replicated_key_provider_factory::on_started(::replica::database& db, service::migration_manager& mm) { + return replicated_key_provider::do_initialize_tables(db, mm); +} + +} diff --git a/ent/encryption/replicated_key_provider.hh b/ent/encryption/replicated_key_provider.hh new file mode 100644 index 0000000000..cd20208e9e --- /dev/null +++ b/ent/encryption/replicated_key_provider.hh @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2015 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include "encryption.hh" + +namespace db { +class extensions; +} + +namespace replica { +class database; +} + +namespace service { +class migration_manager; +} + +namespace encryption { + +class replicated_key_provider_factory : public key_provider_factory { +public: + replicated_key_provider_factory(); + ~replicated_key_provider_factory(); + + shared_ptr get_provider(encryption_context&, const options&) override; + + static void init(db::extensions&); + static future<> on_started(::replica::database&, service::migration_manager&); +}; + +} diff --git a/ent/encryption/symmetric_key.cc b/ent/encryption/symmetric_key.cc new file mode 100644 index 0000000000..7f3a267f9b --- /dev/null +++ b/ent/encryption/symmetric_key.cc @@ -0,0 +1,396 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ +#include +#include +#include + +#include +#include +#include + +#if OPENSSL_VERSION_NUMBER >= (3<<28) +# include +#endif + +#include +#include + +#include "symmetric_key.hh" +#include "utils/hash.hh" + +namespace { +struct openssl_env { + OSSL_PROVIDER* legacy_provider = nullptr; + OSSL_PROVIDER* default_provider = nullptr; + openssl_env() { + OpenSSL_add_all_ciphers(); +#if OPENSSL_VERSION_NUMBER >= (3<<28) + legacy_provider = OSSL_PROVIDER_load(NULL, "legacy"); + default_provider = OSSL_PROVIDER_load(NULL, "default"); +#endif + } + ~openssl_env() { + OSSL_PROVIDER_unload(legacy_provider); + OSSL_PROVIDER_unload(default_provider); + } +}; +static const openssl_env ossl_env; +} + +std::ostream& encryption::operator<<(std::ostream& os, const key_info& info) { + return os << info.alg << ":" << info.len; +} + +static void throw_evp_error(std::string msg) { + auto e = ERR_get_error(); + if (e != 0) { + char buf[512]; + ERR_error_string_n(e, buf, sizeof(buf)); + msg += "(" + std::string(buf) + ")"; + } + throw std::runtime_error(msg); +} + +bool encryption::key_info::compatible(const key_info& rhs) const { + sstring malg, halg; + std::tie(malg, std::ignore, std::ignore) = parse_key_spec(alg); + std::tie(halg, std::ignore, std::ignore) = parse_key_spec(rhs.alg); + if (malg != halg) { + return false; + } + // If lengths differ we need to actual create keys to + // check what the true lengths are. Since openssl and + // java designators count different for DES etc. + if (len != rhs.len) { + symmetric_key k1(*this); + symmetric_key k2(rhs); + if (k1.key().size() != k2.key().size()) { + return false; + } + } + return true; +} + +std::tuple +encryption::parse_key_spec(const sstring& alg) { + static const std::regex alg_exp(R"foo(^(\w+)(?:\/(\w+))?(?:\/(\w+))?$)foo"); + + std::cmatch m; + if (!std::regex_match(alg.begin(), alg.end(), m, alg_exp)) { + throw std::invalid_argument("Invalid algorithm string: " + alg); + } + + auto type = m[1].str(); + auto mode = m[2].str(); + auto padd = m[3].str(); + + std::transform(type.begin(), type.end(), type.begin(), ::tolower); + std::transform(mode.begin(), mode.end(), mode.begin(), ::tolower); + std::transform(padd.begin(), padd.end(), padd.begin(), ::tolower); + + static const std::string padding = "padding"; + if (padd.size() > padding.size() && std::equal(padding.rbegin(), padding.rend(), padd.rbegin())) { + padd.resize(padd.size() - padding.size()); + } + + return std::make_tuple(type, mode, padd); +} + +std::tuple encryption::parse_key_spec_and_validate_defaults(const sstring& alg) { + auto [type, mode, padd] = parse_key_spec(alg); + + // openssl AND kmip server(s?) does not allow missing block mode. so default one. + if (mode.empty()) { + mode = "cbc"; + } + + // OpenSSL only supports one form of padding. We used to just allow + // non-empty string -> pkcs5/pcks7. Better to verify + // (note: pcks5 is sortof a misnomeanor here, as in the Sun world, it + // sort of means "pkcs7 with automatic block size" - which is pretty + // much how things are in the OpenSSL universe as well) + if (padd == "no") { + padd = ""; + } + if (!padd.empty() && padd != "pkcs5" && padd != "pkcs" && padd != "pkcs7") { + throw std::invalid_argument("non-supported padding option: " + padd); + } + + return { type, mode, padd }; +} + +encryption::symmetric_key::symmetric_key(const key_info& info, const bytes& key) + : _ctxt(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free) + , _info(info) + , _key(key) +{ + if (!_ctxt) { + throw std::bad_alloc(); + } + + sstring type, mode, padd; + std::tie(type, mode, padd) = parse_key_spec_and_validate_defaults(info.alg); + + // Note: we are using some types here that are explicitly marked as "unsupported - placeholder" + // in gnutls. + + // camel case vs. dash + if (type == "desede") { + type = "des-ede"; + // and 168-bits desede is ede3 in openssl... + if (info.len > 16*8) { + type = "des-ede3"; + } + } + + auto str = fmt::format("{}-{}-{}", type, info.len, mode); + auto cipher = EVP_get_cipherbyname(str.c_str()); + + if (!cipher) { + str = fmt::format("{}-{}", type, mode); + cipher = EVP_get_cipherbyname(str.c_str()); + } + if (!cipher) { + str = fmt::format("{}-{}", type, info.len); + cipher = EVP_get_cipherbyname(str.c_str()); + } + if (!cipher) { + str = type; + cipher = EVP_get_cipherbyname(str.c_str()); + } + if (!cipher) { + throw_evp_error("Invalid algorithm: " + info.alg); + } + + size_t len = EVP_CIPHER_key_length(cipher); + + if ((_info.len/8) != len) { + if (!EVP_CipherInit_ex(*this, cipher, nullptr, nullptr, nullptr, 0)) { + throw_evp_error("Could not initialize cipher"); + } + auto dlen = _info.len/8; + // Openssl describes des-56 length as 64 (counts parity), + // des-ede-112 as 128 etc... + // do some special casing... + if ((type == "des" || type == "des-ede" || type == "des-ede3") && (dlen & 7) != 0) { + dlen = align_up(dlen, 8u); + } + // if we had to find a cipher without explicit key length (like rc2), + // try to set the key length to the desired strength. + if (!EVP_CIPHER_CTX_set_key_length(*this, dlen)) { + throw_evp_error(fmt::format("Invalid length {} for resolved type {} (wanted {})", len*8, str, _info.len)); + } + + len = EVP_CIPHER_key_length(cipher); + } + + + if (_key.empty()) { + _key.resize(len); + if (!RAND_bytes(reinterpret_cast(_key.data()), _key.size())) { + throw_evp_error(fmt::format("Could not generate key: {}", info.alg)); + } + } + if (_key.size() < len) { + throw std::invalid_argument(fmt::format("Invalid key data length {} for resolved type {} ({})", _key.size()*8, str, len*8)); + } + + if (!EVP_CipherInit_ex(*this, cipher, nullptr, + reinterpret_cast(_key.data()), nullptr, + 0)) { + throw_evp_error("Could not initialize cipher from key materiel"); + } + + _iv_len = EVP_CIPHER_CTX_iv_length(*this); + _block_size = EVP_CIPHER_CTX_block_size(*this); + _padding = !padd.empty(); + +} + +std::string encryption::symmetric_key::validate_exact_info_result() const { + auto [types, modes, padds] = parse_key_spec(_info.alg); + + auto cipher = EVP_CIPHER_CTX_get0_cipher(*this); + auto len = EVP_CIPHER_key_length(cipher); + auto mode = EVP_CIPHER_get_mode(cipher); + + std::ostringstream ss; + + if (unsigned(len)*8 != align_up(_info.len, 16u)) { + ss << "Length " << len*8 << " differs from requested " << _info.len << std::endl; + } + + static std::unordered_map openssl_modes({ + { EVP_CIPH_ECB_MODE, "ecb" }, + { EVP_CIPH_CBC_MODE, "cbc" }, + { EVP_CIPH_CFB_MODE, "cfb" }, + { EVP_CIPH_OFB_MODE, "ofb" }, + { EVP_CIPH_CTR_MODE, "ctr" }, + { EVP_CIPH_GCM_MODE, "cgm" }, + { EVP_CIPH_CCM_MODE, "ccm" }, + { EVP_CIPH_XTS_MODE, "xts" }, + { EVP_CIPH_WRAP_MODE, "wrap"}, + { EVP_CIPH_OCB_MODE, "ocb" }, + { EVP_CIPH_SIV_MODE, "siv" }, + }); + + auto i = openssl_modes.find(mode); + if (i != openssl_modes.end() && i->second != modes) { + ss << _info << ": " << "Block mode " << i->second << " differers from requested " << modes << std::endl; + } + + if ((!padds.empty() && padds != "no") != _padding) { + ss << _info << ": " << "Padding (" << bool(_padding) << " differs from requested " << padds << std::endl; + } + + return ss.str(); +} + +void encryption::symmetric_key::generate_iv_impl(uint8_t* dst, size_t s) const { + if (s < _iv_len) { + throw std::invalid_argument("Buffer underflow"); + } + if (!RAND_bytes(dst, s)) { + throw_evp_error("Could not generate initialization vector"); + } +} + +void encryption::symmetric_key::transform_unpadded_impl(const uint8_t* input, + size_t input_len, uint8_t* output, const uint8_t* iv, mode m) const { + if (!EVP_CipherInit_ex(*this, nullptr, nullptr, + reinterpret_cast(_key.data()), iv, int(m))) { + throw_evp_error("Could not initialize cipher (transform)"); + } + if (!EVP_CIPHER_CTX_set_padding(*this, 0)) { + throw_evp_error("Could not disable padding"); + } + + if (input_len & (_block_size - 1)) { + throw std::invalid_argument("Data must be aligned to 'blocksize'"); + } + + int outl = 0; + auto res = m == mode::decrypt ? + EVP_DecryptUpdate(*this, output, &outl, input, + int(input_len)) : + EVP_EncryptUpdate(*this, output, &outl, input, + int(input_len)); + + if (!res || outl != int(input_len)) { + throw std::runtime_error("transformation failed"); + } +} + +size_t encryption::symmetric_key::decrypt_impl(const uint8_t* input, + size_t input_len, uint8_t* output, size_t output_len, + const uint8_t* iv) const { + if (!EVP_CipherInit_ex(*this, nullptr, nullptr, + reinterpret_cast(_key.data()), iv, 0)) { + throw_evp_error("Could not initialize cipher (decrypt)"); + } + if (!EVP_CIPHER_CTX_set_padding(*this, int(_padding))) { + throw_evp_error("Could not initialize padding"); + } + + // normal case, caller provides output enough to deal with any padding. + // in padding case, max out size is input_len - 1. + if (input_len <= output_len) { + // one go. + int outl = 0; + int finl = 0; + if (!EVP_DecryptUpdate(*this, output, &outl, input, int(input_len))) { + throw_evp_error("decryption failed"); + } + if (!EVP_DecryptFinal(*this, output + outl, &finl)) { + throw_evp_error("decryption failed"); + } + + return outl + finl; + } + + // meh. must provide block padding. + constexpr size_t local_buf_size = 1024; + + static thread_local std::vector cached_buf; + + if (cached_buf.size() < local_buf_size + _block_size) [[unlikely]] { + cached_buf.resize(local_buf_size + _block_size); + } + + auto buf = cached_buf.data(); + size_t res = 0; + while (input_len) { + auto n = std::min(input_len, local_buf_size); + int outl = 0; + if (!EVP_DecryptUpdate(*this, buf, &outl, input, int(n))) { + throw std::runtime_error("decryption failed"); + } + if (n < local_buf_size) { + // last block + int finl = 0; + if (!EVP_DecryptFinal(*this, buf + outl, &finl)) { + throw std::runtime_error("decryption failed"); + } + outl += finl; + } + if ((res + outl) > output_len) { + throw std::invalid_argument("Output buffer too small"); + } + output = std::copy(buf, buf + outl, output); + res += outl; + input_len -= n; + input += n; + } + + return res; +} + +size_t encryption::symmetric_key::encrypted_size(size_t n) const { + // encryption always adds padding. So if n is multiple of blocksize + // the size is n + blocksize. But if its not, things are "better"... + return _block_size + align_down(n, _block_size); +} + +size_t encryption::symmetric_key::encrypt_impl(const uint8_t* input, + size_t input_len, uint8_t* output, size_t output_len, + const uint8_t* iv) const { + if (output_len < encrypted_size(input_len)) { + throw std::invalid_argument("Insufficient buffer"); + } + + if (!EVP_CipherInit_ex(*this, nullptr, nullptr, + reinterpret_cast(_key.data()), iv, 1)) { + throw_evp_error("Could not initialize cipher (encrypt)"); + } + if (!EVP_CIPHER_CTX_set_padding(*this, int(_padding))) { + throw_evp_error("Could not initialize padding"); + } + + int outl = 0; + int finl = 0; + if (!EVP_EncryptUpdate(*this, output, &outl, input, int(input_len))) { + throw_evp_error("encryption failed"); + } + if (!EVP_EncryptFinal(*this, output + outl, &finl)) { + throw_evp_error("encryption failed"); + } + return outl + finl; +} + +bool encryption::operator==(const key_info& k1, const key_info& k2) { + return k1.alg == k2.alg && k1.len == k2.len; +} + +bool encryption::operator!=(const key_info& k1, const key_info& k2) { + return !(k1 == k2); +} + +size_t encryption::key_info_hash::operator()(const key_info& e) const { + return utils::tuple_hash()(std::tie(e.alg, e.len)); +} diff --git a/ent/encryption/symmetric_key.hh b/ent/encryption/symmetric_key.hh new file mode 100644 index 0000000000..4a752e396f --- /dev/null +++ b/ent/encryption/symmetric_key.hh @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "../../bytes.hh" + +// forward declare openssl evp. +extern "C" { +struct evp_cipher_ctx_st; +} + +namespace encryption { + +struct key_info { + sstring alg; + unsigned len; + + bool compatible(const key_info&) const; +}; + +bool operator==(const key_info& k1, const key_info& k2); +bool operator!=(const key_info& k1, const key_info& k2); +std::ostream& operator<<(std::ostream&, const key_info&); + +struct key_info_hash { + size_t operator()(const key_info& e) const; +}; + +std::tuple parse_key_spec(const sstring&); + +// shared between key & kmip +std::tuple parse_key_spec_and_validate_defaults(const sstring&); + +class symmetric_key { + std::unique_ptr _ctxt; + key_info _info; + bytes _key; + unsigned _iv_len = 0; + unsigned _block_size = 0; + bool _padding = true; + + operator evp_cipher_ctx_st *() const { + return _ctxt.get(); + } + + void generate_iv_impl(uint8_t* dst, size_t) const; + size_t decrypt_impl(const uint8_t* input, size_t input_len, uint8_t* output, + size_t output_len, const uint8_t* iv) const; + size_t encrypt_impl(const uint8_t* input, size_t input_len, uint8_t* output, + size_t output_len, const uint8_t* iv) const; + +public: + symmetric_key(const key_info& info, const bytes& key = { }); + + const key_info& info() const { + return _info; + } + const bytes& key() const { + return _key; + } + size_t iv_len() const { + return _iv_len; + } + size_t block_size() const { + return _block_size; + } + + /** + * Evaluates whether or not the key info provided resulted in + * the exact same result from openssl, i.e. whether the combination + * of alg/block mode/padding etc was actually fully valid (or our + * heuristics have issues) + */ + std::string validate_exact_info_result() const; + + /** + * Write a random IV to dst. Must be iv_len() sized or larger + */ + template + void generate_iv(T* dst, size_t s) const { + static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v); + generate_iv_impl(reinterpret_cast(dst), s); + } + + // returns minimal buffer size required to encrypt n bytes. I.e. + // block alignment + size_t encrypted_size(size_t n) const; + + template + size_t decrypt(const T* input, size_t input_len, V* output, + size_t output_len, const I* iv = nullptr) const { + static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v); + return decrypt_impl(reinterpret_cast(input), input_len, + reinterpret_cast(output), output_len, + reinterpret_cast(iv)); + } + template + size_t encrypt(const T* input, size_t input_len, V* output, + size_t output_len, const I* iv = nullptr) const { + static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v); + return encrypt_impl(reinterpret_cast(input), input_len, + reinterpret_cast(output), output_len, + reinterpret_cast(iv)); + } + + enum class mode { + decrypt, encrypt, + }; + template + void transform_unpadded(mode m, const T* input, size_t input_len, V* output, + const I* iv = nullptr) const { + static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v); + return transform_unpadded_impl(reinterpret_cast(input), + input_len, reinterpret_cast(output), + reinterpret_cast(iv), m); + } + template + void encrypt_unpadded(const T* input, size_t input_len, V* output, + const I* iv = nullptr) const { + static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v); + return transform_unpadded_impl(reinterpret_cast(input), + input_len, reinterpret_cast(output), + reinterpret_cast(iv), mode::encrypt); + } + template + void decrypt_unpadded(const T* input, size_t input_len, V* output, + const I* iv = nullptr) const { + static_assert(sizeof(T) == sizeof(uint8_t) && std::is_integral_v); + return transform_unpadded_impl(reinterpret_cast(input), + input_len, reinterpret_cast(output), + reinterpret_cast(iv), mode::decrypt); + } + +private: + void transform_unpadded_impl(const uint8_t* input, size_t input_len, + uint8_t* output, const uint8_t* iv, mode) const; +}; + +} + +template <> struct fmt::formatter : fmt::ostream_formatter {}; diff --git a/ent/encryption/system_key.cc b/ent/encryption/system_key.cc new file mode 100644 index 0000000000..bb8b845639 --- /dev/null +++ b/ent/encryption/system_key.cc @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2015 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ +#include +#include +#include +#include + +#include +#include + +#include + +#include "symmetric_key.hh" +#include "system_key.hh" + +future<> encryption::system_key::validate() const { + return make_ready_future<>(); +} + +future encryption::system_key::decrypt(const sstring& s) { + auto b = base64_decode(s); + return decrypt(b).then([](bytes b) { + return make_ready_future(sstring(b.begin(), b.end())); + }); +} + +future encryption::system_key::encrypt(const sstring& s) { + return encrypt(bytes(s.begin(), s.end())).then([](bytes b) { + return make_ready_future(base64_encode(b)); + }); +} + +future encryption::system_key::encrypt(const bytes& b) { + return get_key().then([b](shared_ptr k) { + auto i = k->iv_len(); + auto n = k->encrypted_size(b.size()); + bytes res(bytes::initialized_later(), n + i); + k->generate_iv(reinterpret_cast(res.data()), i); + n = k->encrypt(reinterpret_cast(b.data()), b.size() + , reinterpret_cast(res.data()) + i, res.size() - i + , reinterpret_cast(res.data())); + res.resize(n + i); + return make_ready_future(std::move(res)); + }); + +} + +future encryption::system_key::decrypt(const bytes& b) { + return get_key().then([b](shared_ptr k) { + auto i = k->iv_len(); + bytes res(bytes::initialized_later(), b.size() - i); + auto n = k->decrypt(reinterpret_cast(b.data()) + i, + b.size() - i, reinterpret_cast(res.data()), + res.size(), reinterpret_cast(b.data())); + res.resize(n); + return make_ready_future(std::move(res)); + }); +} + diff --git a/ent/encryption/system_key.hh b/ent/encryption/system_key.hh new file mode 100644 index 0000000000..8d6ac32b26 --- /dev/null +++ b/ent/encryption/system_key.hh @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2015 ScyllaDB + * + */ + +/* + * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 + */ + +#pragma once + +#include "encryption.hh" +#include "../../bytes.hh" + +namespace encryption { + +class symmetric_key; + +class system_key { +public: + virtual ~system_key() {} + virtual future> get_key() = 0; + virtual const sstring& name() const = 0; + virtual bool is_local() const = 0; + virtual future<> validate() const; + + future encrypt(const sstring&); + future decrypt(const sstring&); + future encrypt(const bytes&); + future decrypt(const bytes&); +}; + +} +