From 7bd13be3f2b64ee5542d1435e2afdc976a66041e Mon Sep 17 00:00:00 2001 From: Michael Livshin Date: Thu, 22 Apr 2021 20:31:24 +0300 Subject: [PATCH] build: improvements & upgrades to Nix dev environment * Add some more useful stuff to the shell environment, so it actually works for debugging & post-mortem analysis. * Wrap ccache & distcc transparently (distcc will be used unless NODISTCC is set to a non-empty value in the environment; ccache will be used if CCACHE_DIR is not empty). * Package the Scylla Python driver (instead of the C* one). * Catch up to misc build/test requirements (including optional) by requiring or custom-packaging: wasmtime 0.29.0, cxxbridge, pytest-asyncio, liburing. * Build statically-linked zstd in a saner and more idiomatic fashion. * In pure builds (where sources lack Git metadata), derive SCYLLA_RELEASE from source hash. * Refactor things for more parameterization. * Explicitly stub out installPhase (seeing that "nix build" succeeds up to installPhase means we didn't miss any dependencies). * Add flake support. * Add copious comments. Signed-off-by: Michael Livshin --- configure.py | 33 +- default.nix | 220 +++++---- dist/nix/overlay.nix | 25 + .../pkg/custom/ccache-distcc-wrap/default.nix | 67 +++ .../nix/pkg/upstreamable/cxxbridge/Cargo.lock | 447 ++++++++++++++++++ .../pkg/upstreamable/cxxbridge/default.nix | 28 ++ .../upstreamable/python-driver/default.nix | 31 ++ .../nix/pkg/upstreamable/wasmtime/default.nix | 47 ++ flake.lock | 43 ++ flake.nix | 51 ++ shell.nix | 37 +- 11 files changed, 902 insertions(+), 127 deletions(-) mode change 100755 => 100644 default.nix create mode 100644 dist/nix/overlay.nix create mode 100644 dist/nix/pkg/custom/ccache-distcc-wrap/default.nix create mode 100644 dist/nix/pkg/upstreamable/cxxbridge/Cargo.lock create mode 100644 dist/nix/pkg/upstreamable/cxxbridge/default.nix create mode 100644 dist/nix/pkg/upstreamable/python-driver/default.nix create mode 100644 dist/nix/pkg/upstreamable/wasmtime/default.nix create mode 100644 flake.lock create mode 100644 flake.nix mode change 100755 => 100644 shell.nix diff --git a/configure.py b/configure.py index 61f2dc06df..be8ff0d10e 100755 --- a/configure.py +++ b/configure.py @@ -44,32 +44,18 @@ distro_extra_cflags = '' distro_extra_ldflags = '' distro_extra_cmake_args = [] employ_ld_trickery = True +has_wasmtime = False +use_wasmtime_as_library = False # distro-specific setup def distro_setup_nix(): - global os_ids, employ_ld_trickery - global distro_extra_ldflags, distro_extra_cflags, distro_extra_cmake_args - + global os_ids, employ_ld_trickery, has_wasmtime, use_wasmtime_as_library os_ids = ['linux'] employ_ld_trickery = False + has_wasmtime = True + use_wasmtime_as_library = True - libdirs = list(dict.fromkeys(os.environ.get('CMAKE_LIBRARY_PATH').split(':'))) - incdirs = list(dict.fromkeys(os.environ.get('CMAKE_INCLUDE_PATH').split(':'))) - - # add nix {lib,inc}dirs to relevant flags, mimicing nix versions of cmake & autotools. - # also add rpaths to make sure that any built executables can run in place. - distro_extra_ldflags = ' '.join([ '-rpath ' + path + ' -L' + path for path in libdirs ]); - distro_extra_cflags = ' '.join([ '-isystem ' + path for path in incdirs ]) - - # indexers like clangd may or may not know which stdc++ or glibc - # the compiler was configured with, so make the relevant paths - # explicit on each compilation command line: - implicit_cflags = os.environ.get('IMPLICIT_CFLAGS').strip() - distro_extra_cflags += ' ' + implicit_cflags - # also propagate to cmake-built dependencies: - distro_extra_cmake_args = ['-DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES:INTERNAL=' + implicit_cflags] - -if os.environ.get('NIX_BUILD_TOP'): +if os.environ.get('NIX_CC'): distro_setup_nix() # distribution "internationalization", converting package names. @@ -1423,7 +1409,8 @@ if flag_supported(flag='-Wstack-usage=4096', compiler=args.cxx): for mode in modes: modes[mode]['cxxflags'] += f' -Wstack-usage={modes[mode]["stack-usage-threshold"]} -Wno-error=stack-usage=' -has_wasmtime = os.path.isfile('/usr/lib64/libwasmtime.a') and os.path.isdir('/usr/local/include/wasmtime') +if not has_wasmtime: + has_wasmtime = os.path.isfile('/usr/lib64/libwasmtime.a') and os.path.isdir('/usr/local/include/wasmtime') if has_wasmtime: if platform.machine() == 'aarch64': @@ -1755,6 +1742,8 @@ libs = ' '.join([maybe_static(args.staticyamlcpp, '-lyaml-cpp'), '-latomic', '-l ]) if has_wasmtime: print("Found wasmtime dependency, linking with libwasmtime") + if use_wasmtime_as_library: + libs += " -lwasmtime" if not args.staticboost: args.user_cflags += ' -DBOOST_TEST_DYN_LINK' @@ -1936,7 +1925,7 @@ with open(buildfile, 'w') as f: for src in srcs if src.endswith('.cc')] objs.append('$builddir/../utils/arch/powerpc/crc32-vpmsum/crc32.S') - if has_wasmtime: + if has_wasmtime and not use_wasmtime_as_library: objs.append('/usr/lib64/libwasmtime.a') has_thrift = False for dep in deps[binary]: diff --git a/default.nix b/default.nix old mode 100755 new mode 100644 index bc1d3033a9..f54331f292 --- a/default.nix +++ b/default.nix @@ -6,83 +6,122 @@ # # -# * At present this is not very useful for nix-build, just for nix-shell +# * "nix build" is unsupported (or rather supported up to and not including +# installPhase), so basically all this is just for "nix develop" # # * IMPORTANT: to avoid using up ungodly amounts of disk space under -# /nix/store/, make sure the actual build directory is physically -# outside this tree, and make ./build a symlink to it +# /nix/store/ when you are not using flakes, make sure to move the +# actual build directory outside this tree and make ./build a +# symlink to it. Or use flakes (seriously, just use flakes). # -{ - pkgs ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/4cb48cc25622334f17ec6b9bf56e83de0d521fb7.tar.gz") {}, - mode ? "release", - verbose ? false, - useCcache ? false, # can't get this to work, see https://github.com/NixOS/nixpkgs/issues/49894 - testInputsFrom ? (_: []), - gitPkg ? (pkgs: pkgs.gitMinimal), -}: +{ flake ? false +, shell ? false +, pkgs ? import { system = builtins.currentSystem; overlays = [ (import ./dist/nix/overlay.nix ) ]; } +, srcPath ? builtins.path { path = ./.; name = "scylla"; } +, repl ? null +, mode ? "release" +, verbose ? false -with pkgs; let +# shell env will want to add stuff to the environment, and the way +# for it to do so is to pass us a function with this signatire: +, devInputs ? ({ pkgs, llvm }: []) +} @ args: + +let inherit (builtins) baseNameOf fetchurl + head + map match - readFile - toString - trace; + split + trace + ; - antlr3Patched = antlr3.overrideAttrs (_: { - patches = [ - (fetchurl { - url = "https://src.fedoraproject.org/rpms/antlr3/raw/f1bb8d639678047935e1761c3bf3c1c7da8d0f1d/f/0006-antlr3memory.hpp-fix-for-C-20-mode.patch"; - }) - ]; - }); - rapidjsonPatched = rapidjson.overrideAttrs (_: { - patches = [ - (fetchurl { - url = "https://src.fedoraproject.org/rpms/rapidjson/raw/48402da9f19d060ffcd40bf2b2e6987212c58b0c/f/rapidjson-1.1.0-c++20.patch"; - }) - ]; - }); - zstdStatic = zstd.overrideAttrs (_: { - cmakeFlags = [ - "-DZSTD_BUILD_SHARED:BOOL=OFF" - "-DZSTD_BUILD_STATIC:BOOL=ON" - "-DZSTD_PROGRAMS_LINK_SHARED:BOOL=OFF" - "-DZSTD_LEGACY_SUPPORT:BOOL=ON" - "-DZSTD_BUILD_TESTS:BOOL=OFF" - ]; - }); + inherit (import (builtins.fetchTarball { + url = "https://github.com/hercules-ci/gitignore/archive/5b9e0ff9d3b551234b4f3eb3983744fa354b17f1.tar.gz"; + sha256 = "01l4phiqgw9xgaxr6jr456qmww6kzghqrnbc7aiiww3h6db5vw53"; + }) { inherit (pkgs) lib; }) + gitignoreSource; - llvmBundle = llvmPackages_11; + # tests don't like boost17x (which is boost177 at the time of writing) + boost = pkgs.boost175; - stdenv = - if useCcache - then (overrideCC llvmBundle.stdenv (ccacheWrapper.override { cc = llvmBundle.clang; })) - else llvmBundle.stdenv; + # current clang13 cannot compile Scylla with sanitizers: + llvm = pkgs.llvmPackages_12; + # llvm = pkgs.llvmPackages_latest; -in stdenv.mkDerivation { - name = "scylladb"; - nativeBuildInputs = [ + stdenvUnwrapped = llvm.stdenv; + + # define custom ccache- and distcc-aware wrappers for all relevant + # compile drivers (used only in shell env) + cc-wrappers = pkgs.callPackage ./dist/nix/pkg/custom/ccache-distcc-wrap { + cc = stdenvUnwrapped.cc; + clang = llvm.clang; + inherit (pkgs) gcc; + }; + + stdenv = if shell then pkgs.overrideCC stdenvUnwrapped cc-wrappers + else stdenvUnwrapped; + + noNix = path: type: type != "regular" || (match ".*\.nix" path) == null; + src = builtins.filterSource noNix (if flake then srcPath + else gitignoreSource srcPath); + + derive = if shell then pkgs.mkShell.override { inherit stdenv; } + else stdenv.mkDerivation; + +in derive ({ + name = "scylla"; + inherit src; + + # since Scylla build, as it exists, is not cross-capable, the + # nativeBuildInputs/buildInputs distinction below ranges, depending + # on how charitable one feels, from "pedantic" through + # "aspirational" all the way to "cargo cult ritual" -- i.e. not + # expected to be actually correct or verifiable. but it's the + # thought that counts! + nativeBuildInputs = with pkgs; [ ant - antlr3Patched - boost17x.dev + antlr3 + boost + cargo cmake + cxxbridge gcc - (gitPkg pkgs) + openjdk11_headless libtool - llvmBundle.lld + llvm.bintools maven ninja pkg-config - python3 + python2 + (python3.withPackages (ps: with ps; [ + aiohttp + boto3 + colorama + distro + magic + psutil + pyparsing + pytest + pytest-asyncio + pyudev + pyyaml + requests + scylla-driver + setuptools + tabulate + urwid + ])) ragel stow - ]; - buildInputs = [ - antlr3Patched - boost17x + ] ++ (devInputs { inherit pkgs llvm; }); + + buildInputs = with pkgs; [ + antlr3 + boost c-ares cryptopp fmt @@ -91,13 +130,16 @@ in stdenv.mkDerivation { hwloc icu jsoncpp + libidn2 libp11 libsystemtap libtasn1 libunistring + liburing libxfs libxml2 libyamlcpp + llvm.compiler-rt lksctp-tools lua53Packages.lua lz4 @@ -106,61 +148,57 @@ in stdenv.mkDerivation { openssl p11-kit protobuf - python3Packages.cassandra-driver - python3Packages.distro - python3Packages.psutil - python3Packages.pyparsing - python3Packages.pyudev - python3Packages.pyyaml - python3Packages.requests - python3Packages.setuptools - python3Packages.urwid - rapidjsonPatched + rapidjson snappy systemd thrift valgrind + wasmtime xorg.libpciaccess xxHash zlib zstdStatic - ] ++ (testInputsFrom pkgs); + ]; - src = lib.cleanSourceWith { - filter = name: type: - let baseName = baseNameOf (toString name); in - !((type == "symlink" && baseName == "build") || - (type == "directory" && - (baseName == "build" || - baseName == ".cache" || - baseName == ".direnv" || - baseName == ".github" || - baseName == ".pytest_cache" || - baseName == "__pycache__"))); - src = ./.; - }; + JAVA8_HOME = "${pkgs.openjdk8_headless}/lib/openjdk"; + JAVA_HOME = "${pkgs.openjdk11_headless}/lib/openjdk"; + +} +// (if shell then { + + configurePhase = "./configure.py${if verbose then " --verbose" else ""} --disable-dpdk"; + +} else { + + # sha256 of the filtered source tree: + SCYLLA_RELEASE = head (split "-" (baseNameOf src)); postPatch = '' patchShebangs ./configure.py - patchShebangs ./merge-compdb.py patchShebangs ./seastar/scripts/seastar-json2code.py patchShebangs ./seastar/cooking.sh ''; - IMPLICIT_CFLAGS = '' - ${readFile (llvmBundle.stdenv.cc + "/nix-support/libcxx-cxxflags")} ${readFile (llvmBundle.stdenv.cc + "/nix-support/libc-cflags")} - ''; - - configurePhase = '' - ./configure.py ${if verbose then "--verbose " else ""}--mode=${mode} - ''; + configurePhase = "./configure.py${if verbose then " --verbose" else ""} --mode=${mode}"; buildPhase = '' - ${ninja}/bin/ninja build/${mode}/scylla + ${pkgs.ninja}/bin/ninja \ + build/${mode}/scylla \ + build/${mode}/iotune \ + ''; + # build/${mode}/dist/tar/scylla-tools-package.tar.gz \ + # build/${mode}/dist/tar/scylla-jmx-package.tar.gz \ installPhase = '' - mkdir $out - cp -r * $out/ + echo not implemented 1>&2 + exit 1 ''; -} + +}) +// (if !shell || repl == null then {} else { + + REPL = repl; + +}) +) diff --git a/dist/nix/overlay.nix b/dist/nix/overlay.nix new file mode 100644 index 0000000000..b5417acb2a --- /dev/null +++ b/dist/nix/overlay.nix @@ -0,0 +1,25 @@ +nixpkgs: +final: prev: + +let + patched = pkg: patches: + pkg.overrideAttrs (old: { + patches = (old.patches or []) ++ (map final.fetchurl patches); + }); +in { + gdbWithGreenThreadSupport = patched prev.gdb [{ + url = "https://github.com/cmm/gnu-binutils/commit/1c52ca4b27e93e1684c68eeaee44ca3e36648410.patch"; + sha256 = "sha256-3s3KvN70dHMdr7Sx1dtzbZ8S+MynPTN7yCocoGlea2Y="; + }]; + + zstdStatic = final.callPackage "${nixpkgs}/pkgs/tools/compression/zstd" { + static = true; + buildContrib = false; + doCheck = false; + }; + + cxxbridge = final.callPackage ./pkg/upstreamable/cxxbridge { }; + wasmtime = final.callPackage ./pkg/upstreamable/wasmtime { }; + + scylla-driver = final.callPackage ./pkg/upstreamable/python-driver { }; +} diff --git a/dist/nix/pkg/custom/ccache-distcc-wrap/default.nix b/dist/nix/pkg/custom/ccache-distcc-wrap/default.nix new file mode 100644 index 0000000000..22c4924598 --- /dev/null +++ b/dist/nix/pkg/custom/ccache-distcc-wrap/default.nix @@ -0,0 +1,67 @@ +{ pkgs +, lib + +, cc ? pkgs.cc +, gcc ? pkgs.gcc +, clang ? pkgs.clang + +, ... }: + +let + inherit (builtins) + map + match + ; + inherit (pkgs) + ccache + coreutils + distcc + runCommand + stdenv + ; + + wrap = pkg: driver: '' + #! ${stdenv.shell} -e + driver=${pkg}/bin/${driver} + dist_driver="$driver${if ((match "clang.*" driver) != null) then " -Wno-error=unused-command-line-argument" else ""}" + distcc=${distcc}/bin/distcc + ccache=${ccache}/bin/ccache + + export DISTCC_IO_TIMEOUT=1200 # hello, repair/row_level.cc + + wrap= + if [[ -z "$NODISTCC" ]]; then + wrap=d + fi + if [[ -n "$CCACHE_DIR" ]]; then + wrap+=c + fi + + if [[ -z "$wrap" ]]; then + exec $driver "$@" + elif [[ "$wrap" == d ]]; then + exec $distcc $dist_driver "$@" + elif [[ "$wrap" == c ]]; then + exec $ccache $driver "$@" + elif [[ "$wrap" == dc ]]; then + export CCACHE_PREFIX=$distcc + exec $ccache $dist_driver "$@" + else + echo wrapper bug 1>&2 + exit 1 + fi + ''; +in runCommand "distcc-ccache-wrap" { } '' + ${coreutils}/bin/mkdir -p $out/bin + ${lib.concatStrings (map ({pkg, driver}: '' + ${coreutils}/bin/echo ${lib.escapeShellArg (wrap pkg driver)} > $out/bin/${driver} + ${coreutils}/bin/chmod +x $out/bin/${driver} + '') [ + { pkg = gcc; driver = "gcc"; } + { pkg = gcc; driver = "g++"; } + { pkg = clang; driver = "clang"; } + { pkg = clang; driver = "clang++"; } + { pkg = cc; driver = "cc"; } + { pkg = cc; driver = "c++"; } + ])} +'' diff --git a/dist/nix/pkg/upstreamable/cxxbridge/Cargo.lock b/dist/nix/pkg/upstreamable/cxxbridge/Cargo.lock new file mode 100644 index 0000000000..ef6b18eba3 --- /dev/null +++ b/dist/nix/pkg/upstreamable/cxxbridge/Cargo.lock @@ -0,0 +1,447 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clang-ast" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c380e0de48337007dfc91b09eb75f567f5f08209437857fbaabbaf2e77e830" +dependencies = [ + "serde", +] + +[[package]] +name = "clap" +version = "3.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" +dependencies = [ + "bitflags", + "clap_lex", + "indexmap", + "strsim", + "textwrap", +] + +[[package]] +name = "clap_lex" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cxx" +version = "1.0.68" +dependencies = [ + "cc", + "cxx-build", + "cxx-gen", + "cxx-test-suite", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", + "rustversion", + "trybuild", +] + +[[package]] +name = "cxx-build" +version = "1.0.68" +dependencies = [ + "cc", + "codespan-reporting", + "cxx-gen", + "once_cell", + "pkg-config", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxx-gen" +version = "0.7.68" +dependencies = [ + "codespan-reporting", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cxx-test-suite" +version = "0.0.0" +dependencies = [ + "cxx", + "cxx-build", + "cxxbridge-flags", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.68" +dependencies = [ + "clap", + "codespan-reporting", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.68" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.68" +dependencies = [ + "clang-ast", + "cxx", + "flate2", + "memmap", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", +] + +[[package]] +name = "demo" +version = "0.0.0" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "dissimilar" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c97b9233581d84b8e1e689cdd3a47b6f69770084fc246e86a7f78b0d9c1d4a5" + +[[package]] +name = "flate2" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "indexmap" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + +[[package]] +name = "jobserver" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +dependencies = [ + "libc", +] + +[[package]] +name = "libc" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" + +[[package]] +name = "link-cplusplus" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cae2cd7ba2f3f63938b9c724475dfb7b9861b545a90324476324ed21dbc8c8" +dependencies = [ + "cc", +] + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "miniz_oxide" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" +dependencies = [ + "adler", +] + +[[package]] +name = "once_cell" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" + +[[package]] +name = "os_str_bytes" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435" + +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + +[[package]] +name = "proc-macro2" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" + +[[package]] +name = "ryu" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" + +[[package]] +name = "scratch" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96311ef4a16462c757bb6a39152c40f58f31cd2602a40fceb937e2bc34e6cbab" + +[[package]] +name = "serde" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a07e33e919ebcd69113d5be0e4d70c5707004ff45188910106854f38b960df4a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" + +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + +[[package]] +name = "trybuild" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc92f558afb6d1d7c6f175eb8d615b8ef49c227543e68e19c123d4ee43d8a7d" +dependencies = [ + "dissimilar", + "glob", + "once_cell", + "serde", + "serde_derive", + "serde_json", + "termcolor", + "toml", +] + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/dist/nix/pkg/upstreamable/cxxbridge/default.nix b/dist/nix/pkg/upstreamable/cxxbridge/default.nix new file mode 100644 index 0000000000..0f58104194 --- /dev/null +++ b/dist/nix/pkg/upstreamable/cxxbridge/default.nix @@ -0,0 +1,28 @@ +{ pkgs +}: + +let + inherit (pkgs) + fetchFromGitHub + rustPlatform + ; + + version = "1.0.68"; + lockFile = ./Cargo.lock; + +in rustPlatform.buildRustPackage { + pname = "cxxbridge-cmd"; + inherit version; + + src = fetchFromGitHub { + owner = "dtolnay"; + repo = "cxx"; + rev = "${version}"; + sha256 = "sha256-DdcbPcxTGJ5rJUJxQR3YBHbe9g3JjgbP/htJqWerRlI="; + }; + + cargoLock = { inherit lockFile; }; + postPatch = "cp ${lockFile} Cargo.lock"; + + cargoBuildFlags = [ "--package cxxbridge-cmd" ]; +} diff --git a/dist/nix/pkg/upstreamable/python-driver/default.nix b/dist/nix/pkg/upstreamable/python-driver/default.nix new file mode 100644 index 0000000000..429bb9b7d3 --- /dev/null +++ b/dist/nix/pkg/upstreamable/python-driver/default.nix @@ -0,0 +1,31 @@ +{ stdenv +, lib +, fetchFromGitHub +, python3Packages +, libev +}: + +let + version = "3.25.4-scylla"; +in python3Packages.buildPythonPackage { + pname = "scylla-driver"; + inherit version; + + # pypi tarball doesn't include tests + src = fetchFromGitHub { + owner = "scylladb"; + repo = "python-driver"; + rev = version; + sha256 = "sha256-LIPZ4sY/wrhmy+kpFUwBvgvbJoXanQLkzMd5Iv0X2mc="; + }; + + postPatch = '' + substituteInPlace setup.py --replace 'geomet>=0.1,<0.3' 'geomet' + ''; + + nativeBuildInputs = with python3Packages; [ cython ]; + buildInputs = [ libev ]; + propagatedBuildInputs = with python3Packages; [ six geomet ]; + + doCheck = false; +} diff --git a/dist/nix/pkg/upstreamable/wasmtime/default.nix b/dist/nix/pkg/upstreamable/wasmtime/default.nix new file mode 100644 index 0000000000..74a6b5487d --- /dev/null +++ b/dist/nix/pkg/upstreamable/wasmtime/default.nix @@ -0,0 +1,47 @@ +{ pkgs +}: + +let + inherit (pkgs) + cmake + fetchFromGitHub + python3 + rustPlatform + ; + + llvm = pkgs.llvmPackages_latest; + clang = llvm.clang; + + pname = "wasmtime"; + version = "0.29.0"; + +in rustPlatform.buildRustPackage { + inherit pname version; + src = fetchFromGitHub { + owner = "bytecodealliance"; + repo = pname; + rev = "v${version}"; + sha256 = "sha256-qmME9zI2vSFQlJWhve7dqAPh8O/6WclWTwO9Pcdvoj8="; + fetchSubmodules = true; + }; + + cargoSha256 = "sha256-xtx1iCtZ9rRJec6Q7ywBM3ov9z0YJMcxz7K59Ox6DKk="; + + nativeBuildInputs = [ python3 cmake clang ]; + buildInputs = [ llvm.libclang ]; + LIBCLANG_PATH = "${llvm.libclang.lib}/lib"; + + cargoBuildFlags = [ "--package wasmtime-c-api" ]; + + # cargo does not install the C(++) headers + postInstall = '' + install -d -m744 $out/include/wasmtime + install -m644 $src/crates/c-api/include/*.h $out/include + install -m644 $src/crates/c-api/include/wasmtime/*.h $out/include/wasmtime + install -m644 $src/crates/c-api/wasm-c-api/include/* $out/include + ''; + + # Scylla has to use this version (later ones change APIs), and it + # happens not to pass its own tests, so: + doCheck = false; +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..3eb173cf07 --- /dev/null +++ b/flake.lock @@ -0,0 +1,43 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1656928814, + "narHash": "sha256-RIFfgBuKz6Hp89yRr7+NR5tzIAbn52h8vT6vXkYjZoM=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "7e2a3b3dfd9af950a856d66b0a7d01e3c18aa249", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1657532632, + "narHash": "sha256-uEwe1CAQxb0eOWIwxQ3zpMs/llshirlrCXZQz6vkYQU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "85deee6d6c8127d360096a5caa0aeb876b976496", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..321575e147 --- /dev/null +++ b/flake.nix @@ -0,0 +1,51 @@ +{ + description = "Monstrously Fast + Scalable NoSQL"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils } @ inputs: { + overlays.default = import ./dist/nix/overlay.nix nixpkgs; + + lib = { + _attrs = system: let + pkgs = import nixpkgs { + inherit system; + overlays = [ self.overlays.default ]; + }; + + repl = pkgs.writeText "repl" '' + let + self = builtins.getFlake (toString ${self.outPath}); + attrs = self.lib._attrs "${system}"; + in { + inherit self; + inherit (attrs) pkgs; + } + ''; + + args = { + flake = true; + srcPath = "${self}"; + inherit pkgs repl; + }; + + package = import ./default.nixpkgs args; + devShell = import ./shell.nix args; + in { + inherit pkgs args package devShell; + }; + }; + } + // (flake-utils.lib.eachDefaultSystem (system: let + packageName = "scylla"; + attrs = self.lib._attrs system; + in { + packages.${packageName} = attrs.package; + defaultPackage = self.packages.${system}.${packageName}; + + inherit (attrs) devShell; + })); +} diff --git a/shell.nix b/shell.nix old mode 100755 new mode 100644 index ebb29f3ab4..770fee58c7 --- a/shell.nix +++ b/shell.nix @@ -4,18 +4,27 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later -{ - pkgs ? null, - mode ? "dev", - useCcache ? false -}: -import ./default.nix ({ - inherit mode useCcache; - testInputsFrom = pkgs: with pkgs; [ - python3Packages.boto3 - python3Packages.colorama - python3Packages.pytest +args: + +import ./default.nix (args // { + shell = true; + + devInputs = { pkgs, llvm }: with pkgs; [ + # for impure building + ccache + distcc + + # for debugging + binutils # addr2line etc. + elfutils + + gdbWithGreenThreadSupport + + llvm.llvm + lz4 # coredumps on modern Systemd installations are lz4-compressed + + # etc + diffutils + colordiff ]; - gitPkg = pkgs: pkgs.gitFull; -} // -(if pkgs != null then { inherit pkgs; } else {})) +})