dist: change nomerges=1 on block devices during fstrim execution

We have problem to run fstrim with nomerges=2, so we need to change
the parameter to 1 during fstrim execution.
To do this, this fix changes follow things:
 - revert dropping scylla_fstrim on Ubuntu 16.04/CentOS
 - disable distribution provided fstrim script
 - enable scylla_fstrim on all distributions
 - introduce --set-nomerges on scylla-blocktune
 - scylla_fstrim call scylla-blocktune by following order:
   - 'scylla-blocktune --set-nomerges 1'
   - 'fstrim' for each devices
   - 'scylla-blocktune --set-nomerges 2'

Fixes #2649

Signed-off-by: Takuya ASADA <syuu@scylladb.com>
Message-Id: <1501531393-21109-1-git-send-email-syuu@scylladb.com>

dist/common/scripts/scylla_fstrim

@@ -44,11 +44,14 @@ def main():
         mountpoints.add(find_mount_point(d))
 
     mountpoints.add(find_mount_point(cfg['commitlog_directory']))
+    # workaround of #2649
+    subprocess.call(["/usr/lib/scylla/scylla-blocktune", "--set-nomerges", "1"])
     for d in mountpoints:
         print("Running fstrim on {} ...".format(d))
         res = subprocess.call(["fstrim", d])
         if res != 0:
             print("{} doesn't support TRIM.".format(d))
+    subprocess.call(["/usr/lib/scylla/scylla-blocktune", "--set-nomerges", "2"])
 
 if __name__ == "__main__":
     main()

dist/common/scripts/scylla_fstrim_setup

@@ -20,16 +20,11 @@
 
 . /usr/lib/scylla/scylla_lib.sh
 
-#
-# We need to setup fstrim using different scripts for each distribution,
-# as follows:
-#
-# - CentOS 7/RHEL 7/Fedora: use fstrim.timer that included the distribution
-# - Debian 8: use scylla-fstrim via /etc/cron.d
-# - Ubuntu 14.04: use scylla-fstrim via /etc/cron.d
-# - Ubuntu 16.04: use /etc/cron.weekly/fstrim that included the distribution
-#
+# We need to disable distribution provided fstrim scripts for each distribution
 
 if is_redhat_variant; then
-    systemctl enable fstrim.timer
+    systemctl disable fstrim.timer
+fi
+if [ "$ID" = "ubuntu" ] && [ -f /etc/cron.weekly/fstrim ]; then
+	mv -fv /etc/cron.weekly/fstrim /etc/cron.weekly/fstrim.old    
 fi

dist/common/scripts/scylla_setup

@@ -14,7 +14,7 @@ if [ "`id -u`" -ne 0 ]; then
 fi
 
 print_usage() {
-    echo "scylla_setup --disks /dev/hda,/dev/hdb... --nic eth0 --ntp-domain centos --ami --setup-nic --developer-mode --no-ec2-check --no-kernel-check --no-verify-package --no-enable-service --no-selinux-setup --no-bootparam-setup --no-ntp-setup --no-raid-setup --no-coredump-setup --no-sysconfig-setup --no-cpuscaling-setup --no-fstrim-setup"
+    echo "scylla_setup --disks /dev/hda,/dev/hdb... --nic eth0 --ntp-domain centos --ami --setup-nic --developer-mode --no-kernel-check --no-verify-package --no-enable-service --no-selinux-setup --no-bootparam-setup --no-ntp-setup --no-raid-setup --no-coredump-setup --no-sysconfig-setup --no-cpuscaling-setup --no-fstrim-setup"
     echo "  --disks			specify disks for RAID"
     echo "  --nic				specify NIC"
     echo "  --ntp-domain			specify NTP domain"
@@ -35,9 +35,7 @@ print_usage() {
     echo "  --no-version-check		skip daily version check"
     echo "  --no-node-exporter        do not install the node exporter"
     echo "  --no-cpuscaling-setup      skip cpu scaling setup"
-    if is_redhat_variant; then
-        echo "  --no-fstrim-setup           skip fstrim setup"
-    fi
+    echo "  --no-fstrim-setup           skip fstrim setup"
     exit 1
 }
 
@@ -286,6 +284,7 @@ if [ $ENABLE_SERVICE -eq 1 ]; then
     if is_systemd; then
         systemctl enable scylla-server.service
         systemctl enable collectd.service
+        systemctl unmask scylla-fstrim.timer
     elif is_gentoo_variant; then
         rc-update add scylla-server default
         rc-update add collectd default
@@ -490,14 +489,12 @@ if [ $CPUSCALING_SETUP -eq 1 ]; then
     run_setup_script "CPU scaling" /usr/lib/scylla/scylla_cpuscaling_setup
 fi
 
-if is_redhat_variant; then
-    if [ $INTERACTIVE -eq 1 ]; then
-        interactive_ask_service "Do you want to enable fstrim service?" "Answer yes to run fstrim on your SSD. Answer no to do nothing." "yes" &&:
-        FSTRIM_SETUP=$?
-    fi
-    if [ $FSTRIM_SETUP -eq 1 ]; then
-        run_setup_script "fstrim" /usr/lib/scylla/scylla_fstrim_setup
-    fi
+if [ $INTERACTIVE -eq 1 ]; then
+    interactive_ask_service "Do you want to enable fstrim service?" "Answer yes to run fstrim on your SSD. Answer no to do nothing." "yes" &&:
+    FSTRIM_SETUP=$?
+fi
+if [ $FSTRIM_SETUP -eq 1 ]; then
+    run_setup_script "fstrim" /usr/lib/scylla/scylla_fstrim_setup
 fi
 
 echo "ScyllaDB setup finished."

dist/common/systemd/scylla-fstrim.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Scylla fstrim
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=/usr/lib/scylla/scylla_fstrim
+
+[Install]
+WantedBy=multi-user.target

dist/common/systemd/scylla-fstrim.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run Scylla fstrim daily
+After=scylla-server.service
+BindsTo=scylla-server.service
+
+[Timer]
+OnCalendar=Sat *-*-* 00:00:00
+RandomizedDelaySec=86400
+
+[Install]
+WantedBy=timers.target

dist/debian/build_deb.sh

@@ -144,9 +144,9 @@ if [ "$TARGET" = "jessie" ]; then
     sed -i -e "s#@@INSTALL@@##g" debian/scylla-server.install
     sed -i -e "s#@@HKDOTTIMER_D@@#dist/common/systemd/scylla-housekeeping-daily.timer /lib/systemd/system#g" debian/scylla-server.install
     sed -i -e "s#@@HKDOTTIMER_R@@#dist/common/systemd/scylla-housekeeping-restart.timer /lib/systemd/system#g" debian/scylla-server.install
+    sed -i -e "s#@@FTDOTTIMER@@#dist/common/systemd/scylla-fstrim.timer /lib/systemd/system#g" debian/scylla-server.install
     sed -i -e "s#@@SYSCTL@@#dist/debian/sysctl.d/99-scylla.conf etc/sysctl.d#g" debian/scylla-server.install
     sed -i -e "s#@@SCRIPTS_SAVE_COREDUMP@@#dist/debian/scripts/scylla_save_coredump usr/lib/scylla#g" debian/scylla-server.install
-    sed -i -e "s#@@SCRIPTS_FSTRIM@@#dist/debian/scripts/scylla_fstrim usr/lib/scylla#g" debian/scylla-server.install
     sed -i -e "s#@@SCRIPTS_DELAY_FSTRIM@@#dist/debian/scripts/scylla_delay_fstrim usr/lib/scylla#g" debian/scylla-server.install
 elif [ "$TARGET" = "stretch" ] || [ "$TARGET" = "buster" ] || [ "$TARGET" = "sid" ]; then
     cp dist/debian/scylla-server.cron.d debian/
@@ -160,7 +160,6 @@ elif [ "$TARGET" = "stretch" ] || [ "$TARGET" = "buster" ] || [ "$TARGET" = "sid
     sed -i -e "s#@@HKDOTTIMER_R@@#dist/common/systemd/scylla-housekeeping-restart.timer /lib/systemd/system#g" debian/scylla-server.install
     sed -i -e "s#@@SYSCTL@@#dist/debian/sysctl.d/99-scylla.conf etc/sysctl.d#g" debian/scylla-server.install
     sed -i -e "s#@@SCRIPTS_SAVE_COREDUMP@@#dist/debian/scripts/scylla_save_coredump usr/lib/scylla#g" debian/scylla-server.install
-    sed -i -e "s#@@SCRIPTS_FSTRIM@@#dist/debian/scripts/scylla_fstrim usr/lib/scylla#g" debian/scylla-server.install
     sed -i -e "s#@@SCRIPTS_DELAY_FSTRIM@@#dist/debian/scripts/scylla_delay_fstrim usr/lib/scylla#g" debian/scylla-server.install
 elif [ "$TARGET" = "trusty" ]; then
     cp dist/debian/scylla-server.cron.d debian/
@@ -172,9 +171,9 @@ elif [ "$TARGET" = "trusty" ]; then
     sed -i -e "s#@@INSTALL@@#dist/debian/sudoers.d/scylla etc/sudoers.d#g" debian/scylla-server.install
     sed -i -e "s#@@HKDOTTIMER_D@@##g" debian/scylla-server.install
     sed -i -e "s#@@HKDOTTIMER_R@@##g" debian/scylla-server.install
+    sed -i -e "s#@@FTDOTTIMER@@##g" debian/scylla-server.install
     sed -i -e "s#@@SYSCTL@@#dist/debian/sysctl.d/99-scylla.conf etc/sysctl.d#g" debian/scylla-server.install
     sed -i -e "s#@@SCRIPTS_SAVE_COREDUMP@@#dist/debian/scripts/scylla_save_coredump usr/lib/scylla#g" debian/scylla-server.install
-    sed -i -e "s#@@SCRIPTS_FSTRIM@@#dist/debian/scripts/scylla_fstrim usr/lib/scylla#g" debian/scylla-server.install
     sed -i -e "s#@@SCRIPTS_DELAY_FSTRIM@@#dist/debian/scripts/scylla_delay_fstrim usr/lib/scylla#g" debian/scylla-server.install
 elif [ "$TARGET" = "xenial" ] || [ "$TARGET" = "yakkety" ] || [ "$TARGET" = "zesty" ] || [ "$TARGET" = "artful" ]; then
     sed -i -e "s/@@REVISION@@/0ubuntu1/g" debian/changelog
@@ -185,9 +184,9 @@ elif [ "$TARGET" = "xenial" ] || [ "$TARGET" = "yakkety" ] || [ "$TARGET" = "zes
     sed -i -e "s#@@INSTALL@@##g" debian/scylla-server.install
     sed -i -e "s#@@HKDOTTIMER_D@@#dist/common/systemd/scylla-housekeeping-daily.timer /lib/systemd/system#g" debian/scylla-server.install
     sed -i -e "s#@@HKDOTTIMER_R@@#dist/common/systemd/scylla-housekeeping-restart.timer /lib/systemd/system#g" debian/scylla-server.install
+    sed -i -e "s#@@FTDOTTIMER@@#dist/common/systemd/scylla-fstrim.timer /lib/systemd/system#g" debian/scylla-server.install
     sed -i -e "s#@@SYSCTL@@##g" debian/scylla-server.install
     sed -i -e "s#@@SCRIPTS_SAVE_COREDUMP@@##g" debian/scylla-server.install
-    sed -i -e "s#@@SCRIPTS_FSTRIM@@##g" debian/scylla-server.install
     sed -i -e "s#@@SCRIPTS_DELAY_FSTRIM@@##g" debian/scylla-server.install
 else
     echo "Unknown distribution: $TARGET"
@@ -201,6 +200,7 @@ cp dist/common/systemd/scylla-server.service.in debian/scylla-server.service
 sed -i -e "s#@@SYSCONFDIR@@#/etc/default#g" debian/scylla-server.service
 cp dist/common/systemd/scylla-housekeeping-daily.service debian/scylla-server.scylla-housekeeping-daily.service
 cp dist/common/systemd/scylla-housekeeping-restart.service debian/scylla-server.scylla-housekeeping-restart.service
+cp dist/common/systemd/scylla-fstrim.service debian/scylla-server.scylla-fstrim.service
 cp dist/common/systemd/node-exporter.service debian/scylla-server.node-exporter.service
 
 if [ $REBUILD -eq 1 ]; then

dist/debian/rules.in

@@ -15,6 +15,7 @@ override_dh_installinit:
 	dh_installinit --no-start @@DH_INSTALLINIT@@
 	dh_installinit --no-start --name scylla-housekeeping-daily @@DH_INSTALLINIT@@
 	dh_installinit --no-start --name scylla-housekeeping-restart @@DH_INSTALLINIT@@
+	dh_installinit --no-start --name scylla-fstrim @@DH_INSTALLINIT@@
 	dh_installinit --no-start --name node-exporter @@DH_INSTALLINIT@@
 
 override_dh_installcron:

dist/debian/scylla-server.install.in

@@ -17,8 +17,8 @@ dist/common/sbin/* usr/sbin
 @@ADDHKCFG@@
 @@HKDOTTIMER_D@@
 @@HKDOTTIMER_R@@
+@@FTDOTTIMER@@
 @@INSTALL@@
 @@SYSCTL@@
 @@SCRIPTS_SAVE_COREDUMP@@
-@@SCRIPTS_FSTRIM@@
 @@SCRIPTS_DELAY_FSTRIM@@

dist/redhat/scylla.spec.in

@@ -210,6 +210,7 @@ rm -rf $RPM_BUILD_ROOT
 %{_prefix}/lib/scylla/scylla_kernel_check
 %{_prefix}/lib/scylla/scylla_cpuset_setup
 %{_prefix}/lib/scylla/scylla_cpuscaling_setup
+%{_prefix}/lib/scylla/scylla_fstrim
 %{_prefix}/lib/scylla/scylla_fstrim_setup
 %{_prefix}/lib/scylla/posix_net_conf.sh
 %{_prefix}/lib/scylla/perftune.py

scylla-blocktune

@@ -49,7 +49,7 @@ tuned_blockdevs = set()
 
 # tune a blockdevice (sysfs node); updates I/O scheduler
 # and merge behavior.  Tunes dependent devices
-def tune_blockdev(path):
+def tune_blockdev(path, nomerges):
     from os.path import join, exists, dirname, realpath
     path = realpath(path)
     print('tuning {}'.format(path))
@@ -58,37 +58,42 @@ def tune_blockdev(path):
     tuned_blockdevs.add(path)
     def check_sched(current):
         return current == 'none' or '[noop]' in current
-    tune_path(join(path, 'queue', 'scheduler'), 'noop', check_sched)
-    tune_path(join(path, 'queue', 'nomerges'), '2')
+    if not nomerges:
+        tune_path(join(path, 'queue', 'scheduler'), 'noop', check_sched)
+        tune_path(join(path, 'queue', 'nomerges'), '2')
+    else:
+        tune_path(join(path, 'queue', 'nomerges'), nomerges)
     slaves = join(path, 'slaves')
     if exists(slaves):
         for slave in os.listdir(slaves):
-            tune_blockdev(join(slaves, slave))
+            tune_blockdev(join(slaves, slave), nomerges)
     if exists(join(path, 'partition')):
-        tune_blockdev(dirname(path))
+        tune_blockdev(dirname(path), nomerges)
 
 # tunes a /dev/foo blockdev
-def tune_dev(path):
+def tune_dev(path, nomerges):
     dev = os.stat(path).st_rdev
     devfile = '/sys/dev/block/{}:{}'.format(dev // 256, dev % 256)
-    tune_blockdev(devfile)
+    tune_blockdev(devfile, nomerges)
 
 # tunes a filesystem
 # FIXME: btrfs
-def tune_fs(path):
+def tune_fs(path, nomerges):
     dev = os.stat(path).st_dev
     devfile = '/sys/dev/block/{}:{}'.format(dev // 256, dev % 256)
-    tune_blockdev(devfile)
+    tune_blockdev(devfile, nomerges)
 
 # tunes all filesystems referenced from a scylla.yaml
-def tune_yaml(path):
+def tune_yaml(path, nomerges):
     import yaml
     y = yaml.load(open(path))
     for fs in y['data_file_directories']:
-        tune_fs(fs)
-    tune_fs(y['commitlog_directory'])
+        tune_fs(fs, nomerges)
+    tune_fs(y['commitlog_directory'], nomerges)
 
 ap = argparse.ArgumentParser('Tune filesystems for ScyllaDB')
+ap.add_argument('--set-nomerges', metavar='VAL', dest='nomerges',
+                help='Overwrite nomerges parameter')
 ap.add_argument('--filesystem', metavar='PATH', action='append', dest='fs', default=[],
                 help='Tune filesystem containing PATH')
 ap.add_argument('--dev', metavar='PATH', action='append', dest='dev', default=[],
@@ -99,11 +104,11 @@ ap.add_argument('--config', metavar='YAML', action='append', dest='yaml', defaul
 args = ap.parse_args()
 
 if not args.yaml and not args.fs and not args.dev:
-    tune_yaml('/etc/scylla/scylla.yaml')
+    tune_yaml('/etc/scylla/scylla.yaml', args.nomerges)
 else:
     for yaml in args.yaml:
-        tune_yaml(yaml)
+        tune_yaml(yaml, args.nomerges)
     for fs in args.fs:
-        tune_fs(fs)
+        tune_fs(fs, args.nomerges)
     for dev in args.dev:
-        tune_dev(dev)
+        tune_dev(dev, args.nomerges)