scylla_post_install.sh: Add CAP_PERFMON to AmbientCapabilities

Add CAP_PERFMON to AmbientCapabilities in capabilities.conf, to enable
perf_event based stall detector in Seastar.

However, on Debian/Ubuntu CAP_PERFMON with non-root user does not work
because it sets kernel.perf_event_paranoid=4 which disallow all non-root
user access.
(On Debian it kernel.perf_event_paranoid=3)
So we need to configure kernel.perf_event_paranoid=2 on these distros.
see: https://askubuntu.com/questions/1400874/what-does-perf-paranoia-level-four-do

Also, CAP_PERFMON is only available on linux-5.8+, older kernel does not
have this capability.
To enable older kernel environment such as CentOS7, we need to configure
kernel.perf_event_paranoid=1 to allow non-root user access even without
the capability.

Fixes #15743

Closes scylladb/scylladb#16070

dist/common/kernel_conf/post_install.sh

@@ -7,6 +7,28 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 #
 
+version_ge() {
+    [  "$2" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
+}
+
+KERNEL_VER=$(uname -r)
+
+if ! version_ge $KERNEL_VER 5.8; then
+    # On older kernel environment, we have to relax perf_event_paranoid setting
+    # since there is no CAP_PERFMON.
+    PERF_EVENT_PARANOID=1
+elif [ $(cat /proc/sys/kernel/perf_event_paranoid) -ge 3  ]; then
+    # On Debian/Ubuntu, it deny access from non-root even with CAP_PERFMON
+    # It requires to set perf_event_paranoid=2 to use CAP_PERFMON with non-root
+    PERF_EVENT_PARANOID=2
+fi
+if [ -n "$PERF_EVENT_PARANOID" ]; then
+    cat << EOS > /etc/sysctl.d/99-scylla-perfevent.conf
+kernel.perf_event_paranoid = $PERF_EVENT_PARANOID
+EOS
+    sysctl -p /etc/sysctl.d/99-scylla-perfevent.conf
+fi
+
 if [ ! -d /run/systemd/system ]; then
     exit 0
 fi

dist/debian/debian/scylla-kernel-conf.postrm

@@ -2,6 +2,14 @@
 
 set -e
 
+case "$1" in
+    purge|remove)
+        if [ "$1" = "purge" ]; then
+            rm -f /etc/sysctl.d/99-scylla-perfevent.conf
+        fi
+        ;;
+esac
+
 if [ -d /run/systemd/system ]; then
     systemctl --system daemon-reload >/dev/null || true
 fi

dist/redhat/scylla.spec

@@ -228,6 +228,7 @@ fi
 %{_sysctldir}/*.conf
 %{_unitdir}/scylla-tune-sched.service
 /opt/scylladb/kernel_conf/*
+%ghost /etc/sysctl.d/99-scylla-perfevent.conf
 
 
 %package node-exporter

scylla_post_install.sh

@@ -11,6 +11,10 @@ if [ ! -d /run/systemd/system ]; then
     exit 0
 fi
 
+version_ge() {
+    [  "$2" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
+}
+
 # Install capabilities.conf when AmbientCapabilities supported
 . /etc/os-release
 
@@ -26,13 +30,20 @@ fi
 
 AMB_SUPPORT=`grep -c ^CapAmb: /proc/self/status`
 
+KERNEL_VER=$(uname -r)
+
 # AmbientCapabilities supported from v229 but it backported to v219-33 on RHEL7
 if [ $SYSTEMD_VER -ge 229 ] || [[ $SYSTEMD_VER -eq 219 && $SYSTEMD_REL -ge 33 ]]; then
     if [ $AMB_SUPPORT -eq 1 ]; then
+        AMB_CAPABILITIES="CAP_SYS_NICE CAP_IPC_LOCK"
+        # CAP_PERFMON is only available on linux-5.8+
+        if version_ge $KERNEL_VER 5.8; then
+            AMB_CAPABILITIES="$AMB_CAPABILITIES CAP_PERFMON"
+        fi
         mkdir -p /etc/systemd/system/scylla-server.service.d/
         cat << EOS > /etc/systemd/system/scylla-server.service.d/capabilities.conf
 [Service]
-AmbientCapabilities=CAP_SYS_NICE CAP_IPC_LOCK
+AmbientCapabilities=$AMB_CAPABILITIES
 EOS
     fi
 fi