dist: tune tcp_mem to 3% of total memory in scylla-kernel-conf package

tcp_mem defaults to 9% of total memory. ScyllaDB defaults to 93%. The
sum is more than 100%.

Fix by tuning tcp_mem to 3% of total memory.

Fixes https://scylladb.atlassian.net/browse/SCYLLADB-734

Closes scylladb/scylladb#28700

dist/common/kernel_conf/post_install.sh

@@ -31,6 +31,23 @@ EOS
     sysctl -p /etc/sysctl.d/99-scylla-perfevent.conf
 fi
 
+# Tune tcp_mem to max out at 3% of total system memory.
+# Seastar defaults to allocating 93% of physical memory. The kernel's default
+# allocation for TCP is ~9%. This adds up to 102%. Reduce the TCP allocation
+# to 3% to avoid OOM.
+PAGE_SIZE=$(getconf PAGE_SIZE)
+TOTAL_MEM_KB=$(sed -n 's/^MemTotal:[[:space:]]*\([0-9]*\).*/\1/p' /proc/meminfo)
+TOTAL_MEM_BYTES=$((TOTAL_MEM_KB * 1024))
+TCP_MEM_MAX=$((TOTAL_MEM_BYTES * 3 / 100))
+TCP_MEM_MAX_PAGES=$((TCP_MEM_MAX / PAGE_SIZE))
+TCP_MEM_MID_PAGES=$((TCP_MEM_MAX * 2 / 3 / PAGE_SIZE))
+TCP_MEM_MIN_PAGES=$((TCP_MEM_MAX / 2 / PAGE_SIZE))
+cat << EOS > /etc/sysctl.d/99-scylla-tcp.conf
+# Scylla: limit TCP memory to 3% of total system memory
+net.ipv4.tcp_mem = $TCP_MEM_MIN_PAGES $TCP_MEM_MID_PAGES $TCP_MEM_MAX_PAGES
+EOS
+sysctl -p /etc/sysctl.d/99-scylla-tcp.conf || :
+
 if [ ! -d /run/systemd/system ]; then
     exit 0
 fi

dist/debian/control.template

@@ -39,7 +39,7 @@ Description: debugging symbols for %{product}-server
 
 Package: %{product}-kernel-conf
 Architecture: any
-Depends: procps
+Depends: procps, sed
 Replaces: scylla-enterprise-kernel-conf (<< 2025.1.0~)
 Breaks: scylla-enterprise-kernel-conf (<< 2025.1.0~)
 Description: Scylla kernel tuning configuration

dist/debian/debian/scylla-kernel-conf.postrm

@@ -6,6 +6,7 @@ case "$1" in
     purge|remove)
         if [ "$1" = "purge" ]; then
             rm -f /etc/sysctl.d/99-scylla-perfevent.conf
+            rm -f /etc/sysctl.d/99-scylla-tcp.conf
         fi
         ;;
 esac

dist/redhat/scylla.spec

@@ -186,7 +186,7 @@ This package contains the main scylla configuration file.
 %package kernel-conf
 Group:          Applications/Databases
 Summary:        Scylla configuration package for the Linux kernel
-Requires:       kmod
+Requires:       kmod sed
 # tuned overwrites our sysctl settings
 Obsoletes:      tuned >= 2.11.0
 Provides:       scylla-enterprise-kernel-conf = %{version}-%{release}
@@ -220,6 +220,7 @@ fi
 %{_unitdir}/scylla-tune-sched.service
 /opt/scylladb/kernel_conf/*
 %ghost /etc/sysctl.d/99-scylla-perfevent.conf
+%ghost /etc/sysctl.d/99-scylla-tcp.conf
 
 
 %package node-exporter

unified/uninstall.sh

@@ -64,6 +64,7 @@ if ! $nonroot; then
     rm -fv "$retc"/bash_completion.d/nodetool-completion
     rm -fv "$retc"/supervisord.d/scylla-*.ini
     rm -fv "$rusr"/lib/sysctl.d/99-scylla-*.conf
+    rm -fv "$retc"/sysctl.d/99-scylla-tcp.conf
     rm -fv "$rusr"/bin/{scylla,iotune,scyllatop}
     rm -fv "$rusr"/sbin/{scylla_*setup,node_exporter_install,node_health_check,scylla_ec2_check,scylla_kernel_check}
     find "$rusr"/lib/scylla -type l -exec rm -fv {} \;