From 8635d24424dab59e4054209ec3f2616a246a9282 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 25 Dec 2022 14:12:39 +0200 Subject: [PATCH] build: drop abseil submodule, replace with distribution abseil This lets us carry fewer things and rely on the distribution for maintenance. The frozen toolchain is updated. Incidental updates include clang 15.0.6, and pytest that doesn't need workarounds. Closes #12397 --- .gitmodules | 3 -- CMakeLists.txt | 12 +------- abseil | 1 - configure.py | 68 +++++------------------------------------ install-dependencies.sh | 2 ++ tools/toolchain/image | 2 +- 6 files changed, 12 insertions(+), 76 deletions(-) delete mode 160000 abseil diff --git a/.gitmodules b/.gitmodules index 46720091b2..e5dc17eded 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,9 +6,6 @@ path = swagger-ui url = ../scylla-swagger-ui ignore = dirty -[submodule "abseil"] - path = abseil - url = ../abseil-cpp [submodule "scylla-jmx"] path = tools/jmx url = ../scylla-jmx diff --git a/CMakeLists.txt b/CMakeLists.txt index acdd49d73d..81ced0fe8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,22 +42,13 @@ set(Seastar_CXX_FLAGS ${cxx_coro_flag} ${target_arch_flag} CACHE INTERNAL "" FOR set(Seastar_CXX_DIALECT gnu++20 CACHE INTERNAL "" FORCE) add_subdirectory(seastar) -add_subdirectory(abseil) -# Exclude absl::strerror from the default "all" target since it's not -# used in Scylla build and, moreover, makes use of deprecated glibc APIs, -# such as sys_nerr, which are not exposed from "stdio.h" since glibc 2.32, -# which happens to be the case for recent Fedora distribution versions. -# -# Need to use the internal "absl_strerror" target name instead of namespaced -# variant because `set_target_properties` does not understand the latter form, -# unfortunately. -set_target_properties(absl_strerror PROPERTIES EXCLUDE_FROM_ALL TRUE) # System libraries dependencies find_package(Boost COMPONENTS filesystem program_options system thread regex REQUIRED) find_package(Lua REQUIRED) find_package(ZLIB REQUIRED) find_package(ICU COMPONENTS uc REQUIRED) +find_package(Abseil REQUIRED) set(scylla_build_dir "${CMAKE_BINARY_DIR}/build/${BUILD_TYPE}") set(scylla_gen_build_dir "${scylla_build_dir}/gen") @@ -746,7 +737,6 @@ target_compile_definitions(scylla PRIVATE XXH_PRIVATE_API HAVE_LZ4_COMPRESS_DEFA target_include_directories(scylla PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}" libdeflate - abseil "${scylla_gen_build_dir}") ### diff --git a/abseil b/abseil deleted file mode 160000 index 4e5ff1559c..0000000000 --- a/abseil +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4e5ff1559ca3bd7bb777a1c48106464cb656e041 diff --git a/configure.py b/configure.py index e9716d5b4f..6c5ae1b506 100755 --- a/configure.py +++ b/configure.py @@ -1667,50 +1667,12 @@ for mode in build_modes: modes[mode]['seastar_cflags'] = seastar_pc_cflags modes[mode]['seastar_libs'] = seastar_pc_libs -def configure_abseil(build_dir, mode, mode_config): - abseil_build_dir = os.path.join(build_dir, mode, 'abseil') +abseil_pkgs = [ + 'absl_raw_hash_set', + 'absl_hash', +] - abseil_cflags = seastar_cflags + ' ' + modes[mode]['cxx_ld_flags'] - cmake_mode = mode_config['cmake_build_type'] - abseil_cmake_args = [ - '-DCMAKE_BUILD_TYPE={}'.format(cmake_mode), - '-DCMAKE_INSTALL_PREFIX={}'.format(build_dir + '/inst'), # just to avoid a warning from absl - '-DCMAKE_C_COMPILER={}'.format(args.cc), - '-DCMAKE_CXX_COMPILER={}'.format(args.cxx), - '-DCMAKE_CXX_FLAGS_{}={}'.format(cmake_mode.upper(), abseil_cflags), - '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', - '-DCMAKE_CXX_STANDARD=20', - '-DABSL_PROPAGATE_CXX_STD=ON', - ] + distro_extra_cmake_args - - abseil_cmd = ['cmake', '-G', 'Ninja', real_relpath('abseil', abseil_build_dir)] + abseil_cmake_args - - os.makedirs(abseil_build_dir, exist_ok=True) - subprocess.check_call(abseil_cmd, shell=False, cwd=abseil_build_dir) - -abseil_libs = ['absl/' + lib for lib in [ - 'container/libabsl_hashtablez_sampler.a', - 'container/libabsl_raw_hash_set.a', - 'synchronization/libabsl_synchronization.a', - 'synchronization/libabsl_graphcycles_internal.a', - 'debugging/libabsl_stacktrace.a', - 'debugging/libabsl_symbolize.a', - 'debugging/libabsl_debugging_internal.a', - 'debugging/libabsl_demangle_internal.a', - 'time/libabsl_time.a', - 'time/libabsl_time_zone.a', - 'numeric/libabsl_int128.a', - 'hash/libabsl_hash.a', - 'hash/libabsl_city.a', - 'hash/libabsl_low_level_hash.a', - 'base/libabsl_malloc_internal.a', - 'base/libabsl_spinlock_wait.a', - 'base/libabsl_base.a', - 'base/libabsl_raw_logging_internal.a', - 'profiling/libabsl_exponential_biased.a', - 'strings/libabsl_strings.a', - 'strings/libabsl_strings_internal.a', - 'base/libabsl_throw_delegate.a']] +pkgs += abseil_pkgs args.user_cflags += " " + pkg_config('jsoncpp', '--cflags') args.user_cflags += ' -march=' + args.target @@ -1745,7 +1707,6 @@ if any(filter(thrift_version.startswith, thrift_boost_versions)): for pkg in pkgs: args.user_cflags += ' ' + pkg_config(pkg, '--cflags') libs += ' ' + pkg_config(pkg, '--libs') -args.user_cflags += ' -isystem abseil' user_cflags = args.user_cflags + ' -fvisibility=hidden' user_ldflags = args.user_ldflags + ' -fvisibility=hidden' if args.staticcxx: @@ -1767,10 +1728,6 @@ if args.ragel_exec: else: ragel_exec = "ragel" -if not args.dist_only: - for mode, mode_config in build_modes.items(): - configure_abseil(outdir, mode, mode_config) - with open(buildfile, 'w') as f: f.write(textwrap.dedent('''\ configure_args = {configure_args} @@ -1931,9 +1888,6 @@ with open(buildfile, 'w') as f: if binary.endswith('.a'): f.write('build $builddir/{}/{}: ar.{} {}\n'.format(mode, binary, mode, str.join(' ', objs))) else: - objs.extend(['$builddir/' + mode + '/' + artifact for artifact in [ - 'abseil/' + x for x in abseil_libs - ]]) if binary in tests: local_libs = '$seastar_libs_{} $libs'.format(mode) if binary in pure_boost_tests: @@ -2124,12 +2078,6 @@ with open(buildfile, 'w') as f: f.write(f'build $builddir/{mode}/dist/tar/{scylla_product}-unified-package-{scylla_version}-{scylla_release}.tar.gz: copy $builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz\n') f.write(f'build $builddir/{mode}/dist/tar/{scylla_product}-unified-{arch}-package-{scylla_version}-{scylla_release}.tar.gz: copy $builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz\n') - for lib in abseil_libs: - f.write('build $builddir/{mode}/abseil/{lib}: ninja $builddir/{mode}/abseil/build.ninja\n'.format(**locals())) - f.write(' pool = submodule_pool\n') - f.write(' subdir = $builddir/{mode}/abseil\n'.format(**locals())) - f.write(' target = {lib}\n'.format(**locals())) - checkheaders_mode = 'dev' if 'dev' in modes else modes.keys()[0] f.write('build checkheaders: phony || {}\n'.format(' '.join(['$builddir/{}/{}.o'.format(checkheaders_mode, hh) for hh in headers]))) @@ -2240,7 +2188,7 @@ with open(buildfile, 'w') as f: description = List configured modes build mode_list: mode_list default {modes_list} - ''').format(modes_list=' '.join(default_modes), build_ninja_list=' '.join([f'build/{mode}/{dir}/build.ninja' for mode in build_modes for dir in ['seastar', 'abseil']]), **globals())) + ''').format(modes_list=' '.join(default_modes), build_ninja_list=' '.join([f'build/{mode}/{dir}/build.ninja' for mode in build_modes for dir in ['seastar']]), **globals())) unit_test_list = set(test for test in build_artifacts if test in set(tests)) f.write(textwrap.dedent('''\ rule unit_test_list @@ -2269,7 +2217,7 @@ with open(buildfile, 'w') as f: compdb = 'compile_commands.json' # per-mode compdbs are built by taking the relevant entries from the # output of "ninja -t compdb" and combining them with the CMake-made -# compdbs for Seastar and Abseil in the relevant mode. +# compdbs for Seastar in the relevant mode. # # "ninja -t compdb" output has to be filtered because # - it contains rules for all selected modes, and several entries for @@ -2284,7 +2232,7 @@ with tempfile.NamedTemporaryFile() as ninja_compdb: # build mode-specific compdbs for mode in selected_modes: mode_out = outdir + '/' + mode - submodule_compdbs = [mode_out + '/' + submodule + '/' + compdb for submodule in ['abseil', 'seastar']] + submodule_compdbs = [mode_out + '/' + submodule + '/' + compdb for submodule in ['seastar']] with open(mode_out + '/' + compdb, 'w+b') as combined_mode_specific_compdb: subprocess.run(['./scripts/merge-compdb.py', 'build/' + mode, ninja_compdb.name] + submodule_compdbs, stdout=combined_mode_specific_compdb) diff --git a/install-dependencies.sh b/install-dependencies.sh index eed0c5ede4..f39555f497 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -46,6 +46,7 @@ debian_base_packages=( libunistring-dev libzstd-dev libdeflate-dev + libabsl-dev ) fedora_packages=( @@ -61,6 +62,7 @@ fedora_packages=( snappy-devel libdeflate-devel systemd-devel + abseil-cpp-devel git python sudo diff --git a/tools/toolchain/image b/tools/toolchain/image index c4a92439a7..c590692209 100644 --- a/tools/toolchain/image +++ b/tools/toolchain/image @@ -1 +1 @@ -docker.io/scylladb/scylla-toolchain:fedora-37-20221116 \ No newline at end of file +docker.io/scylladb/scylla-toolchain:fedora-37-20221225 \ No newline at end of file