diff --git a/.gitmodules b/.gitmodules index 339ec2d76c..de5224d6ea 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,6 +6,9 @@ path = swagger-ui url = ../scylla-swagger-ui ignore = dirty +[submodule "abseil"] + path = abseil + url = ../abseil-cpp [submodule "scylla-jmx"] path = tools/jmx url = ../scylla-jmx diff --git a/CMakeLists.txt b/CMakeLists.txt index f9094fdf2d..3a56007c88 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,33 @@ set(Seastar_EXCLUDE_APPS_FROM_ALL ON CACHE BOOL "" FORCE) set(Seastar_EXCLUDE_TESTS_FROM_ALL ON CACHE BOOL "" FORCE) set(Seastar_UNUSED_RESULT_ERROR ON CACHE BOOL "" FORCE) add_subdirectory(seastar) +set(ABSL_PROPAGATE_CXX_STD ON CACHE BOOL "" FORCE) + +find_package(Sanitizers QUIET) +set(sanitizer_cxx_flags + $<$,Debug;Sanitize>:$;$>) +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(ABSL_GCC_FLAGS ${sanitizer_cxx_flags}) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(ABSL_LLVM_FLAGS ${sanitizer_cxx_flags}) +endif() +set(ABSL_DEFAULT_LINKOPTS + $<$,Debug;Sanitize>:$;$>) +add_subdirectory(abseil) +add_library(absl-headers INTERFACE) +target_include_directories(absl-headers INTERFACE + "${PROJECT_SOURCE_DIR}/abseil") +add_library(absl::headers ALIAS absl-headers) + +# Exclude absl::strerror from the default "all" target since it's not +# used in Scylla build and, moreover, makes use of deprecated glibc APIs, +# such as sys_nerr, which are not exposed from "stdio.h" since glibc 2.32, +# which happens to be the case for recent Fedora distribution versions. +# +# Need to use the internal "absl_strerror" target name instead of namespaced +# variant because `set_target_properties` does not understand the latter form, +# unfortunately. +set_target_properties(absl_strerror PROPERTIES EXCLUDE_FROM_ALL TRUE) # System libraries dependencies find_package(Boost REQUIRED @@ -66,7 +93,6 @@ target_link_libraries(Boost::regex find_package(Lua REQUIRED) find_package(ZLIB REQUIRED) find_package(ICU COMPONENTS uc i18n REQUIRED) -find_package(absl COMPONENTS hash raw_hash_set REQUIRED) find_package(fmt 9.0.0 REQUIRED) find_package(libdeflate REQUIRED) find_package(libxcrypt REQUIRED) @@ -124,6 +150,8 @@ target_sources(scylla-main target_link_libraries(scylla-main PRIVATE db + absl::headers + absl::btree absl::hash absl::raw_hash_set Seastar::seastar @@ -236,6 +264,7 @@ target_link_libraries(scylla PRIVATE target_link_libraries(scylla PRIVATE seastar + absl::headers Boost::program_options) target_include_directories(scylla PRIVATE diff --git a/abseil b/abseil new file mode 160000 index 0000000000..d7aaad83b4 --- /dev/null +++ b/abseil @@ -0,0 +1 @@ +Subproject commit d7aaad83b488fd62bd51c81ecf16cd938532cc0a diff --git a/alternator/CMakeLists.txt b/alternator/CMakeLists.txt index 9863d176e1..d067dcd948 100644 --- a/alternator/CMakeLists.txt +++ b/alternator/CMakeLists.txt @@ -27,7 +27,8 @@ target_link_libraries(alternator cql3 idl Seastar::seastar - xxHash::xxhash) + xxHash::xxhash + absl::headers) check_headers(check-headers alternator GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh) diff --git a/api/CMakeLists.txt b/api/CMakeLists.txt index a3c6d58253..5db3601e0e 100644 --- a/api/CMakeLists.txt +++ b/api/CMakeLists.txt @@ -72,7 +72,8 @@ target_link_libraries(api idl wasmtime_bindings Seastar::seastar - xxHash::xxhash) + xxHash::xxhash + absl::headers) check_headers(check-headers api GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh) diff --git a/auth/CMakeLists.txt b/auth/CMakeLists.txt index adfac14e64..8c7cd26ac4 100644 --- a/auth/CMakeLists.txt +++ b/auth/CMakeLists.txt @@ -30,6 +30,7 @@ target_link_libraries(scylla_auth Seastar::seastar xxHash::xxhash PRIVATE + absl::headers cql3 idl wasmtime_bindings diff --git a/cdc/CMakeLists.txt b/cdc/CMakeLists.txt index 6086889cad..18b05100f0 100644 --- a/cdc/CMakeLists.txt +++ b/cdc/CMakeLists.txt @@ -11,6 +11,7 @@ target_include_directories(cdc ${CMAKE_SOURCE_DIR}) target_link_libraries(cdc PUBLIC + absl::headers Seastar::seastar xxHash::xxhash PRIVATE diff --git a/configure.py b/configure.py index 84f4918fdf..ad12cee142 100755 --- a/configure.py +++ b/configure.py @@ -282,7 +282,7 @@ def generate_compdb(compdb, ninja, buildfile, modes): # build mode-specific compdbs for mode in modes: mode_out = outdir + '/' + mode - submodule_compdbs = [mode_out + '/' + submodule + '/' + compdb for submodule in ['seastar']] + submodule_compdbs = [mode_out + '/' + submodule + '/' + compdb for submodule in ['seastar', 'abseil']] with open(mode_out + '/' + compdb, 'w+b') as combined_mode_specific_compdb: subprocess.run(['./scripts/merge-compdb.py', outdir + '/' + mode, ninja_compdb.name] + submodule_compdbs, stdout=combined_mode_specific_compdb) @@ -1750,6 +1750,58 @@ def configure_seastar(build_dir, mode, mode_config): subprocess.check_call(seastar_cmd, shell=False, cwd=cmake_dir) +def configure_abseil(build_dir, mode, mode_config): + # for sanitizer cflags + seastar_flags = query_seastar_flags(f'{outdir}/{mode}/seastar/seastar.pc', + mode_config['build_seastar_shared_libs'], + args.staticcxx) + seastar_cflags = seastar_flags['seastar_cflags'] + + abseil_build_dir = os.path.join(build_dir, mode, 'abseil') + + abseil_cflags = seastar_cflags + ' ' + modes[mode]['cxx_ld_flags'] + cmake_mode = mode_config['cmake_build_type'] + abseil_cmake_args = [ + '-DCMAKE_BUILD_TYPE={}'.format(cmake_mode), + '-DCMAKE_INSTALL_PREFIX={}'.format(build_dir + '/inst'), # just to avoid a warning from absl + '-DCMAKE_C_COMPILER={}'.format(args.cc), + '-DCMAKE_CXX_COMPILER={}'.format(args.cxx), + '-DCMAKE_CXX_FLAGS_{}={}'.format(cmake_mode.upper(), abseil_cflags), + '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', + '-DCMAKE_CXX_STANDARD=20', + '-DABSL_PROPAGATE_CXX_STD=ON', + ] + + abseil_cmd = ['cmake', '-G', 'Ninja', real_relpath('abseil', abseil_build_dir)] + abseil_cmake_args + + os.makedirs(abseil_build_dir, exist_ok=True) + subprocess.check_call(abseil_cmd, shell=False, cwd=abseil_build_dir) + +abseil_libs = ['absl/' + lib for lib in [ + 'container/libabsl_hashtablez_sampler.a', + 'container/libabsl_raw_hash_set.a', + 'synchronization/libabsl_synchronization.a', + 'synchronization/libabsl_graphcycles_internal.a', + 'debugging/libabsl_stacktrace.a', + 'debugging/libabsl_symbolize.a', + 'debugging/libabsl_debugging_internal.a', + 'debugging/libabsl_demangle_internal.a', + 'time/libabsl_time.a', + 'time/libabsl_time_zone.a', + 'numeric/libabsl_int128.a', + 'hash/libabsl_hash.a', + 'hash/libabsl_city.a', + 'hash/libabsl_low_level_hash.a', + 'base/libabsl_malloc_internal.a', + 'base/libabsl_spinlock_wait.a', + 'base/libabsl_base.a', + 'base/libabsl_raw_logging_internal.a', + 'profiling/libabsl_exponential_biased.a', + 'strings/libabsl_strings.a', + 'strings/libabsl_strings_internal.a', + 'base/libabsl_throw_delegate.a']] + + def query_seastar_flags(pc_file, use_shared_libs, link_static_cxx=False): if use_shared_libs: opt = '--shared' @@ -1769,9 +1821,7 @@ def query_seastar_flags(pc_file, use_shared_libs, link_static_cxx=False): 'seastar_testing_libs': testing_libs} pkgs = ['libsystemd', - 'jsoncpp', - 'absl_raw_hash_set', - 'absl_hash'] + 'jsoncpp'] # Lua can be provided by lua53 package on Debian-like # systems and by Lua on others. pkgs.append('lua53' if have_pkg('lua53') else 'lua') @@ -2057,6 +2107,9 @@ def write_build_file(f, if has_thrift: local_libs += ' ' + maybe_static(args.staticthrift, '-lthrift') local_libs += ' ' + maybe_static(args.staticboost, '-lboost_system') + objs.extend(['$builddir/' + mode + '/' + artifact for artifact in [ + 'abseil/' + x for x in abseil_libs + ]]) if binary in tests: if binary in pure_boost_tests: local_libs += ' ' + maybe_static(args.staticboost, '-lboost_unit_test_framework') @@ -2265,6 +2318,12 @@ def write_build_file(f, f.write(f'build $builddir/{mode}/dist/tar/{scylla_product}-unified-package-{scylla_version}-{scylla_release}.tar.gz: copy $builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz\n') f.write(f'build $builddir/{mode}/dist/tar/{scylla_product}-unified-{arch}-package-{scylla_version}-{scylla_release}.tar.gz: copy $builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz\n') + for lib in abseil_libs: + f.write('build $builddir/{mode}/abseil/{lib}: ninja $builddir/{mode}/abseil/build.ninja\n'.format(**locals())) + f.write(' pool = submodule_pool\n') + f.write(' subdir = $builddir/{mode}/abseil\n'.format(**locals())) + f.write(' target = {lib}\n'.format(**locals())) + checkheaders_mode = 'dev' if 'dev' in modes else modes.keys()[0] f.write('build checkheaders: phony || {}\n'.format(' '.join(['$builddir/{}/{}.o'.format(checkheaders_mode, hh) for hh in headers]))) @@ -2394,7 +2453,7 @@ def write_build_file(f, description = List configured modes build mode_list: mode_list default {modes_list} - ''').format(modes_list=' '.join(default_modes), build_ninja_list=' '.join([f'{outdir}/{mode}/{dir}/build.ninja' for mode in build_modes for dir in ['seastar']]), **globals())) + ''').format(modes_list=' '.join(default_modes), build_ninja_list=' '.join([f'{outdir}/{mode}/{dir}/build.ninja' for mode in build_modes for dir in ['seastar', 'abseil']]), **globals())) unit_test_list = set(test for test in build_artifacts if test in set(tests)) f.write(textwrap.dedent('''\ rule unit_test_list @@ -2438,11 +2497,14 @@ def create_build_system(args): mode_config['per_src_extra_cxxflags']['release.cc'] = ' '.join(get_release_cxxflags(scylla_version, scylla_release)) if not args.dist_only: + global user_cflags, libs # args.buildfile builds seastar with the rules of # {outdir}/{mode}/seastar/build.ninja, and # {outdir}/{mode}/seastar/seastar.pc is queried for building flags for mode, mode_config in build_modes.items(): configure_seastar(outdir, mode, mode_config) + configure_abseil(outdir, mode, mode_config) + user_cflags += ' -isystem abseil' for mode, mode_config in build_modes.items(): mode_config.update(query_seastar_flags(f'{outdir}/{mode}/seastar/seastar.pc', @@ -2496,7 +2558,6 @@ def configure_using_cmake(args): if args.staticboost: settings['Boost_USE_STATIC_LIBS'] = 'ON' - source_dir = os.path.realpath(os.path.dirname(__file__)) build_dir = os.path.join(source_dir, 'build') diff --git a/cql3/CMakeLists.txt b/cql3/CMakeLists.txt index 6f843bf83c..f9c72f5a9f 100644 --- a/cql3/CMakeLists.txt +++ b/cql3/CMakeLists.txt @@ -131,6 +131,7 @@ target_link_libraries(cql3 idl wasmtime_bindings Seastar::seastar + absl::headers xxHash::xxhash ANTLR3::antlr3 PRIVATE diff --git a/db/CMakeLists.txt b/db/CMakeLists.txt index a169cdbd00..0a81be423b 100644 --- a/db/CMakeLists.txt +++ b/db/CMakeLists.txt @@ -45,6 +45,7 @@ target_link_libraries(db mutation Seastar::seastar xxHash::xxhash + absl::headers PRIVATE data_dictionary cql3) diff --git a/gms/CMakeLists.txt b/gms/CMakeLists.txt index 9943cfa1de..38b747bc32 100644 --- a/gms/CMakeLists.txt +++ b/gms/CMakeLists.txt @@ -20,7 +20,8 @@ target_link_libraries(gms Seastar::seastar xxHash::xxhash PRIVATE - db) + db + absl::headers) check_headers(check-headers gms GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh) diff --git a/idl/CMakeLists.txt b/idl/CMakeLists.txt index f7e0f787a2..4973239575 100644 --- a/idl/CMakeLists.txt +++ b/idl/CMakeLists.txt @@ -74,6 +74,9 @@ add_custom_target(idl-sources DEPENDS ${idl_sources}) add_library(idl INTERFACE) add_dependencies(idl idl-sources) +target_link_directories(idl + INTERFACE + absl::headers) target_include_directories(idl INTERFACE ${scylla_gen_build_dir}) diff --git a/install-dependencies.sh b/install-dependencies.sh index 74d3b206b3..190e1a2f14 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -48,7 +48,6 @@ debian_base_packages=( libunistring-dev libzstd-dev libdeflate-dev - libabsl-dev librapidxml-dev libcrypto++-dev libxxhash-dev @@ -70,7 +69,6 @@ fedora_packages=( snappy-devel libdeflate-devel systemd-devel - abseil-cpp-devel cryptopp-devel git python diff --git a/lang/CMakeLists.txt b/lang/CMakeLists.txt index 73c85d56e3..060796150c 100644 --- a/lang/CMakeLists.txt +++ b/lang/CMakeLists.txt @@ -16,6 +16,7 @@ target_link_libraries(lang Seastar::seastar xxHash::xxhash PRIVATE + absl::headers ${LUA_LIBRARIES}) check_headers(check-headers lang diff --git a/message/CMakeLists.txt b/message/CMakeLists.txt index a92ffa4f2f..59dd3bc35b 100644 --- a/message/CMakeLists.txt +++ b/message/CMakeLists.txt @@ -10,6 +10,7 @@ target_link_libraries(message PUBLIC gms Seastar::seastar + absl::headers PRIVATE idl) diff --git a/mutation/CMakeLists.txt b/mutation/CMakeLists.txt index 090fa20e9e..8da5783018 100644 --- a/mutation/CMakeLists.txt +++ b/mutation/CMakeLists.txt @@ -21,7 +21,9 @@ target_link_libraries(mutation PUBLIC idl Seastar::seastar - xxHash::xxhash) + xxHash::xxhash + PRIVATE + absl::headers) check_headers(check-headers mutation GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh) diff --git a/raft/CMakeLists.txt b/raft/CMakeLists.txt index 7f84d03aa3..b59058a131 100644 --- a/raft/CMakeLists.txt +++ b/raft/CMakeLists.txt @@ -12,7 +12,9 @@ target_include_directories(raft target_link_libraries(raft PUBLIC Seastar::seastar - xxHash::xxhash) + xxHash::xxhash + PRIVATE + absl::headers) check_headers(check-headers raft GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh) diff --git a/repair/CMakeLists.txt b/repair/CMakeLists.txt index 6ae9e67981..24a0c9add3 100644 --- a/repair/CMakeLists.txt +++ b/repair/CMakeLists.txt @@ -6,10 +6,12 @@ target_sources(repair table_check.cc) target_include_directories(repair PUBLIC + absl::headers ${CMAKE_SOURCE_DIR}) target_link_libraries(repair PUBLIC idl + absl::headers Seastar::seastar xxHash::xxhash) diff --git a/replica/CMakeLists.txt b/replica/CMakeLists.txt index f5e0c9c5e0..3745836ccc 100644 --- a/replica/CMakeLists.txt +++ b/replica/CMakeLists.txt @@ -16,6 +16,7 @@ target_include_directories(replica target_link_libraries(replica PUBLIC db + absl::headers wasmtime_bindings Seastar::seastar xxHash::xxhash diff --git a/schema/CMakeLists.txt b/schema/CMakeLists.txt index a7f325969e..14688f7b7a 100644 --- a/schema/CMakeLists.txt +++ b/schema/CMakeLists.txt @@ -12,7 +12,8 @@ target_link_libraries(schema cql3 idl Seastar::seastar - xxHash::xxhash) + xxHash::xxhash + absl::headers) check_headers(check-headers schema GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh) diff --git a/scylla-gdb.py b/scylla-gdb.py index 44d3f6bcfb..8210cd5311 100755 --- a/scylla-gdb.py +++ b/scylla-gdb.py @@ -589,8 +589,52 @@ class flat_hash_map: return self.__nonzero__() +class absl_container: + # absl_container is the underlying type for flat_hash_map and flat_hash_set + # if we need to print flat_hash_set, we should yield the element of set + # instead of tuple in `__iter__()` + def __init__(self, ref): + self.val = ref + HasInfozShift = 1 + self.size = ref["settings_"]["value"]["size_"] >> HasInfozShift + + def __len__(self): + return self.size + + def __iter__(self): + if self.size == 0: + return + capacity = int(self.val["settings_"]["value"]["capacity_"]) + control = self.val["settings_"]["value"]["control_"] + # for the map the slot_type is std::pair + slot_type = gdb.lookup_type(str(self.val.type.strip_typedefs()) + "::slot_type") + slots = self.val["settings_"]["value"]["slots_"].cast(slot_type.pointer()) + for i in range(capacity): + ctrl_t = int(control[i]) + # if the control is empty or deleted, its value is less than -1, see + # https://github.com/abseil/abseil-cpp/blob/c1e1b47d989978cde8c5a2a219df425b785a0c47/absl/container/internal/raw_hash_set.h#L487-L503 + if ctrl_t == -1: + break + if ctrl_t >= 0: + # NOTE: this only works for flat_hash_map + yield slots[i]['key'], slots[i]['value'] + + def __nonzero__(self): + return self.size > 0 + + def __bool__(self): + return self.size > 0 + + def unordered_map(ref): - return flat_hash_map(ref) if ref.type.name.startswith('flat_hash_map') else std_unordered_map(ref) + if ref.type.name.startswith('flat_hash_map'): + try: + return flat_hash_map(ref) + except gdb.error: + # newer absl container uses a different memory layout + return absl_container(ref) + else: + return std_unordered_map(ref) def std_priority_queue(ref): diff --git a/service/CMakeLists.txt b/service/CMakeLists.txt index 43963b3d90..467efb9b71 100644 --- a/service/CMakeLists.txt +++ b/service/CMakeLists.txt @@ -37,6 +37,7 @@ target_include_directories(service target_link_libraries(service PUBLIC db + absl::headers Seastar::seastar xxHash::xxhash PRIVATE diff --git a/sstables/CMakeLists.txt b/sstables/CMakeLists.txt index f7a52afd09..673f036747 100644 --- a/sstables/CMakeLists.txt +++ b/sstables/CMakeLists.txt @@ -31,6 +31,7 @@ target_link_libraries(sstables PRIVATE readers tracing + absl::headers libdeflate::libdeflate ZLIB::ZLIB)