Merge "add relocatable CLI tools required for scylla setup scripts" from Takuya

"
To make offline installer easier we need to minimize dependencies as
possible.
Python dependencies are already dropped by adding relocatable python3 by
Glauber, now it's time to drop rest of command line tools which used by
scylla setup tools.
(even scripts are converted to python3, it still executes some external
commands, so these commands should be distributed with offline installer)

Note that some of CLI tools haven't added such as NTP and RAID stuff,
since these tools have daemons, not just CLI.
To use such stuff in offline mode, users have to install them manually.
But both NTP setup and RAID setup are optional, users still can run Scylla w/o
them.
"

Toolchain updated to docker.io/scylladb/scylla-toolchain:fedora-29-20190401
for changes in install-dependencies.sh; also updates to gnutls 3.6.7 security
release.

* 'reloc_clitools_v5' of https://github.com/syuu1228/scylla:
  reloc: add relocatable CLI tools for scylla setup scripts
  dist/redhat: drop systemd-libs from dependency
  dist/redhat: drop file from dependency since it seems unused
  dist/redhat: drop pciutils from dependency since it only used in DPDK mode

dist/common/scripts/node_health_check

@@ -17,6 +17,8 @@
 # You should have received a copy of the GNU General Public License
 # along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 
+# When a CLI tool is not installed, use relocatable CLI tool provided by Scylla
+export PATH=$PATH:/opt/scylladb/bin
 
 ##Variables##
 REPORT="./`hostname`-health-check-report.txt"
@@ -143,24 +145,6 @@ else
     echo "--------------------------------------------------"
 fi
 
-
-#Install 'net-tools' pkg, to be used for netstat command#
-echo "Installing 'net-tools' Package (for 'netstat' command)"
-echo "--------------------------------------------------"
-
-if [ "$IS_FEDORA" == "0" ]; then
-    sudo yum install net-tools -y -q
-fi
-
-if [ "$IS_DEBIAN" == "0" ]; then
-#    sudo apt-get update -qq
-    sudo apt-get install net-tools -y | grep already
-fi
-
-if [ "$IS_GENTOO" == "0" ]; then
-    sudo emerge -1uq sys-apps/ethtool sys-apps/net-tools
-fi
-
 #Create dir structure to save output_files#
 echo "--------------------------------------------------"
 echo "Creating Output Files Directory"
@@ -282,7 +266,11 @@ for i in `ls -I lo /sys/class/net/`; do echo "--$i"; cat /sys/class/net/$i/queue
 for i in `ls -I lo /sys/class/net/`; do echo "--$i"; cat /sys/class/net/$i/queues/rx-*/rps_flow_cnt; echo ""; done > $OUTPUT_PATH5/rfs-conf.txt
 ps -elf | grep irqbalance > $OUTPUT_PATH5/irqbalance-conf.txt
 sudo sysctl -a > $OUTPUT_PATH5/sysctl.txt 2>&1
-sudo iptables -L -v > $OUTPUT_PATH5/iptables.txt
+if [ -e /usr/sbin/iptables ]; then
+    sudo iptables -L -v > $OUTPUT_PATH5/iptables.txt
+else
+    touch $OUTPUT_PATH5/iptables.txt
+fi
 netstat -an | grep tcp > $OUTPUT_PATH5/netstat-tcp.txt
 
 

dist/common/scripts/scylla_util.py

@@ -263,15 +263,19 @@ class scylla_cpuinfo:
             return len(self._cpu_data["system"])
 
 
+# When a CLI tool is not installed, use relocatable CLI tool provided by Scylla
+scylla_env = os.environ.copy()
+scylla_env['PATH'] =  scylla_env['PATH'] + ':/opt/scylladb/bin'
+
 def run(cmd, shell=False, silent=False, exception=True):
     stdout = subprocess.DEVNULL if silent else None
     stderr = subprocess.DEVNULL if silent else None
     if not shell:
         cmd = shlex.split(cmd)
     if exception:
-        return subprocess.check_call(cmd, shell=shell, stdout=stdout, stderr=stderr)
+        return subprocess.check_call(cmd, shell=shell, stdout=stdout, stderr=stderr, env=scylla_env)
     else:
-        p = subprocess.Popen(cmd, shell=shell, stdout=stdout, stderr=stderr)
+        p = subprocess.Popen(cmd, shell=shell, stdout=stdout, stderr=stderr, env=scylla_env)
         return p.wait()
 
 
@@ -279,9 +283,9 @@ def out(cmd, shell=False, exception=True):
     if not shell:
         cmd = shlex.split(cmd)
     if exception:
-        return subprocess.check_output(cmd, shell=shell).strip().decode('utf-8')
+        return subprocess.check_output(cmd, shell=shell, env=scylla_env).strip().decode('utf-8')
     else:
-        p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE)
+        p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, env=scylla_env)
         return p.communicate()[0].strip().decode('utf-8')
 
 

dist/redhat/scylla.spec.mustache

@@ -49,7 +49,7 @@ BuildRequires: pystache
 {{#centos7}}
 Requires: kernel >= 3.10.0-514
 {{/centos7}}
-Requires:       {{product}}-conf systemd-libs hwloc curl util-linux pciutils mdadm xfsprogs file {{product}}-python3
+Requires:       {{product}}-conf curl mdadm xfsprogs {{product}}-python3
 Conflicts:      abrt
 AutoReqProv:    no
 

install-dependencies.sh

@@ -61,6 +61,13 @@ fedora_packages=(
     python3-magic
     dnf-utils
     pigz
+    net-tools
+    tar
+    gzip
+    gawk
+    util-linux
+    ethtool
+    hwloc
 )
 
 centos_packages=(

scripts/create-relocatable-package.py

@@ -60,7 +60,15 @@ ap.add_argument('--mode', dest='mode', default='release',
 args = ap.parse_args()
 
 executables = ['build/{}/scylla'.format(args.mode),
-               'build/{}/iotune'.format(args.mode)]
+               'build/{}/iotune'.format(args.mode),
+               '/usr/bin/lscpu',
+               '/usr/bin/gawk',
+               '/usr/bin/gzip',
+               '/usr/sbin/ifconfig',
+               '/usr/sbin/ethtool',
+               '/usr/bin/netstat',
+               '/usr/bin/hwloc-distrib',
+               '/usr/bin/hwloc-calc']
 
 output = args.dest
 

tools/toolchain/image

@@ -1 +1 @@
-docker.io/scylladb/scylla-toolchain:fedora-29-20190319
+docker.io/scylladb/scylla-toolchain:fedora-29-20190401