dist: make p11-kit-trust.so able to work in relocatable package

Currently, our relocatable package doesn't contains p11-kit-trust.so
since it dynamically loaded, not showing on "ldd" results
(Relocatable packaging script finds dependent libraries by "ldd").
So we need to add it on create-relocatable-pacakge.py.

Also, we have two more problems:
1. p11 module load path is defined as "/usr/lib64/pkcs11", not
   referencing to /opt/scylladb/libreloc
   (and also RedHat variants uses different path than Debian variants)

2. ca-trust-source path is configured on build time (on Fedora),
   it compatible with RedHat variants but not compatible with Debian
   variants

To solve these problems, we need to override default p11-kit
configuration.
To do so, we need to add an configuration file to
/opt/scylladb/share/pkcs11/modules/p11-kit-trust.module.
Also, ofcause p11-kit doesn't reference /opt/scylladb by default, we
need to override load path by p11_kit_override_system_files().

On the configuration file, we can specify module load path by "modules: <path>",
and also we can specify ca-trust-source path by "x-init-reservied: paths=<path>".

Fixes scylladb/scylladb#13904

Closes scylladb/scylladb#22302

dist/debian/debian/rules

@@ -19,7 +19,7 @@ override_dh_auto_clean:
 
 override_dh_auto_install:
 	dh_auto_install
-	./scylla/install.sh --packaging --root "$(CURDIR)/debian/tmp" $(install_arg) --sysconfdir "/etc/default"
+	./scylla/install.sh --packaging --root "$(CURDIR)/debian/tmp" $(install_arg) --sysconfdir "/etc/default" --p11-trust-paths /etc/ssl/certs/ca-certificates.crt
 	# don't use default sysconfig file, use Debian version
 	cp scylla/dist/debian/sysconfig/scylla-housekeeping $(CURDIR)/debian/tmp/etc/default/
 

dist/debian/debian/scylla-server.install

@@ -2,6 +2,7 @@ etc/default/scylla-server
 etc/default/scylla-housekeeping
 etc/scylla.d/*.conf
 etc/bash_completion.d/nodetool-completion
+opt/scylladb/share/p11-kit/modules/*
 opt/scylladb/share/doc/scylla/*
 opt/scylladb/share/doc/scylla/licenses/
 usr/lib/systemd/system/*.timer

dist/redhat/scylla.spec

@@ -60,7 +60,7 @@ This package installs all required packages for ScyllaDB,  including
 %if 0%{housekeeping}
 install_arg="--housekeeping"
 %endif
-./install.sh --packaging --root "$RPM_BUILD_ROOT" $install_arg
+./install.sh --packaging --root "$RPM_BUILD_ROOT" --p11-trust-paths /etc/pki/ca-trust/source:/usr/share/pki/ca-trust-source $install_arg
 
 %clean
 rm -rf $RPM_BUILD_ROOT
@@ -111,6 +111,7 @@ ln -sfT /etc/scylla /var/lib/scylla/conf
 %config(noreplace) %{_sysconfdir}/sysconfig/scylla-housekeeping
 %attr(0755,root,root) %dir %{_sysconfdir}/scylla.d
 %config(noreplace) %{_sysconfdir}/scylla.d/*.conf
+/opt/scylladb/share/p11-kit/modules/*
 /opt/scylladb/share/doc/scylla/*
 %{_unitdir}/scylla-fstrim.service
 %{_unitdir}/scylla-housekeeping-daily.service

install.sh

@@ -30,6 +30,7 @@ Options:
   --supervisor             enable supervisor to manage scylla processes
   --supervisor-log-to-stdout logging to stdout on supervisor
   --without-systemd         skip installing systemd units
+  --p11-trust-paths         specify trust path for p11-kit
   --help                   this helpful message
 EOF
     exit 1
@@ -71,6 +72,7 @@ supervisor=false
 supervisor_log_to_stdout=false
 without_systemd=false
 skip_systemd_check=false
+p11_trust_paths=
 
 while [ $# -gt 0 ]; do
     case "$1" in
@@ -121,6 +123,10 @@ while [ $# -gt 0 ]; do
             skip_systemd_check=true
             shift 1
             ;;
+        "--p11-trust-paths")
+            p11_trust_paths="$2"
+            shift 2
+            ;;
         "--help")
             shift 1
 	    print_usage
@@ -232,6 +238,17 @@ check_usermode_support() {
 
 . /etc/os-release
 
+is_redhat_variant() {
+    is_redhat=0
+    for i in $ID $ID_LIKE; do
+        if [ "$i" = "rhel" -o "$i" = "fedora" -o "$i" = "centos" ]; then
+            is_redhat=1
+            break
+        fi
+    done
+    [ $is_redhat -eq 1 ]
+}
+
 is_debian_variant() {
     [ "$ID_LIKE" = "debian" -o "$ID" = "debian" ]
 }
@@ -271,6 +288,30 @@ cd "$(dirname "$0")"
 
 product="$(cat ./SCYLLA-PRODUCT-FILE)"
 
+if [ -z "$p11_trust_paths" ]; then
+    # our package builder is cross-distro, so we cannot detect distro by os-release
+    if $packaging; then
+        echo "Please specify --p11-trust-paths."
+        echo "The path can be get by following command:"
+        echo "  pkg-config --variable p11_trust_paths p11-kit-1"
+        echo
+        print_usage
+    else
+        # for offline installer users we provide default p11-trust-paths
+        if is_redhat_variant; then
+            p11_trust_paths=/etc/pki/ca-trust/source:/usr/share/pki/ca-trust-source
+        elif is_debian_variant; then
+            p11_trust_paths=/etc/ssl/certs/ca-certificates.crt
+        else
+            echo "Please specify --p11-trust-paths."
+            echo "The path can be get by following command:"
+            echo "  pkg-config --variable p11_trust_paths p11-kit-1"
+            echo
+            print_usage
+        fi
+    fi
+fi
+
 if [ -z "$prefix" ]; then
     if $nonroot; then
         prefix=~/scylladb
@@ -303,6 +344,7 @@ if ! $nonroot; then
     rsysconfdir=$(realpath -m "$root/$sysconfdir")
     rusr=$(realpath -m "$root/usr")
     rsystemd=$(realpath -m "$rusr/lib/systemd/system")
+    rshare="$rprefix/share"
     rdoc="$rprefix/share/doc"
     rdata=$(realpath -m "$root/var/lib/scylla")
     rhkdata=$(realpath -m "$root/var/lib/scylla-housekeeping")
@@ -310,6 +352,7 @@ else
     retc="$rprefix/etc"
     rsysconfdir="$rprefix/$sysconfdir"
     rsystemd="$HOME/.config/systemd/user"
+    rshare="$rprefix/share"
     rdoc="$rprefix/share/doc"
     rdata="$rprefix"
 fi
@@ -401,7 +444,7 @@ for file in dist/common/scylla.d/*.conf; do
     installconfig 644 "$file" "$retc"/scylla.d
 done
 
-install -d -m755 "$retc"/scylla "$rprefix/bin" "$rprefix/libexec" "$rprefix/libreloc" "$rprefix/scripts" "$rprefix/bin"
+install -d -m755 "$retc"/scylla "$rprefix/bin" "$rprefix/libexec" "$rprefix/libreloc" "$rprefix/libreloc/pkcs11" "$rprefix/scripts" "$rprefix/bin"
 if ! $without_systemd; then
     install -m644 dist/common/systemd/scylla-fstrim.service -Dt "$rsystemd"
     install -m644 dist/common/systemd/scylla-housekeeping-daily.service -Dt "$rsystemd"
@@ -412,10 +455,16 @@ if ! $without_systemd; then
 fi
 install -m755 seastar/scripts/seastar-cpu-map.sh -Dt "$rprefix"/scripts
 install -m755 seastar/dpdk/usertools/dpdk-devbind.py -Dt "$rprefix"/scripts
-install -m755 libreloc/* -Dt "$rprefix/libreloc"
+for i in $(find libreloc/ -maxdepth 1 -type f); do
+    install -m755 "$i" -Dt "$rprefix/libreloc"
+done
 for lib in libreloc/*; do
     remove_rpath "$rprefix/$lib"
 done
+for i in $(find libreloc/pkcs11/ -maxdepth 1 -type f); do
+    install -m755 "$i" -Dt "$rprefix/libreloc/pkcs11"
+done
+
 # some files in libexec are symlinks, which "install" dereferences
 # use cp -P for the symlinks instead.
 install -m755 libexec/* -Dt "$rprefix/libexec"
@@ -464,6 +513,16 @@ PRODUCT="$product"
 EOS
 chmod 644 "$rprefix"/scripts/scylla_product.py
 
+install -d -m755 "$rshare"/p11-kit/modules
+cat << EOS > "$rshare"/p11-kit/modules/p11-kit-trust.module
+module: $prefix/libreloc/pkcs11/p11-kit-trust.so
+priority: 1
+trust-policy: yes
+x-trust-lookup: pkcs11:library-description=PKCS%2311%20Kit%20Trust%20Module
+disable-in: p11-kit-proxy
+x-init-reserved: paths=$p11_trust_paths
+EOS
+
 if ! $nonroot && ! $without_systemd; then
     install -d -m755 "$retc"/systemd/system/scylla-server.service.d
     install -m644 dist/common/systemd/scylla-server.service.d/dependencies.conf -Dt "$retc"/systemd/system/scylla-server.service.d

main.cc

@@ -125,6 +125,8 @@ extern "C" {
 #include <p11-kit/p11-kit.h>
 }
 
+namespace fs = std::filesystem;
+
 seastar::metrics::metric_groups app_metrics;
 
 using namespace std::chrono_literals;
@@ -754,6 +756,14 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl
         print_starting_message(ac, av, parsed_opts);
     }
 
+    // We have to override p11-kit config path before p11-kit initialization.
+    // And the initialization will invoke on seastar initalization, so it has to
+    // be before app.run()
+    auto scylla_path = fs::read_symlink(fs::path("/proc/self/exe"));
+    auto p11_modules = scylla_path.parent_path().parent_path().append("share/p11-kit/modules");
+    auto p11_modules_str = p11_modules.string<char>();
+    ::p11_kit_override_system_files(NULL, NULL, p11_modules_str.c_str(), NULL, NULL);
+
     sharded<locator::shared_token_metadata> token_metadata;
     sharded<locator::effective_replication_map_factory> erm_factory;
     sharded<service::migration_notifier> mm_notifier;

scripts/create-relocatable-package.py

@@ -110,6 +110,8 @@ for exe in executables:
 
 # manually add libthread_db for debugging thread
 libs.update({'libthread_db.so.1': os.path.realpath('/lib64/libthread_db.so')})
+# manually add p11-kit-trust.so since it will dynamically load
+libs.update({'pkcs11/p11-kit-trust.so': '/lib64/pkcs11/p11-kit-trust.so'})
 
 ld_so = libs['ld.so']
 
@@ -131,6 +133,10 @@ with tempfile.NamedTemporaryFile('w+t') as version_file:
     version_file.flush()
     ar.add(version_file.name, arcname='.relocatable_package_version')
 
+with tempfile.TemporaryDirectory() as tmpdir:
+    os.symlink('./pkcs11/p11-kit-trust.so', f'{tmpdir}/libnssckbi.so')
+    ar.reloc_add(f'{tmpdir}/libnssckbi.so', arcname='libreloc/libnssckbi.so')
+
 for exe in executables_scylla:
     basename = os.path.basename(exe)
     if not args.stripped: