From 7d78515d5bcb64e3d0a78b713401861f503ba8f6 Mon Sep 17 00:00:00 2001 From: Takuya ASADA Date: Fri, 22 Feb 2019 06:06:13 +0900 Subject: [PATCH 1/4] dist/redhat: drop pciutils from dependency since it only used in DPDK mode Since we don't use DPDK mode by default, and the mode is not officially supported, drop pciutils from package dependency. Users who want to use DPDK mode they neeed to install the package manually. --- dist/redhat/scylla.spec.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/redhat/scylla.spec.mustache b/dist/redhat/scylla.spec.mustache index 12abe39c95..629053b2a4 100644 --- a/dist/redhat/scylla.spec.mustache +++ b/dist/redhat/scylla.spec.mustache @@ -49,7 +49,7 @@ BuildRequires: pystache {{#centos7}} Requires: kernel >= 3.10.0-514 {{/centos7}} -Requires: {{product}}-conf systemd-libs hwloc curl util-linux pciutils mdadm xfsprogs file {{product}}-python3 +Requires: {{product}}-conf systemd-libs hwloc curl util-linux mdadm xfsprogs file {{product}}-python3 Conflicts: abrt AutoReqProv: no From a3741b405217df3c73a071da628d2bbd67d376cc Mon Sep 17 00:00:00 2001 From: Takuya ASADA Date: Fri, 22 Feb 2019 19:05:44 +0900 Subject: [PATCH 2/4] dist/redhat: drop file from dependency since it seems unused The pacakge is not used in our script anymore, drop it. --- dist/redhat/scylla.spec.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/redhat/scylla.spec.mustache b/dist/redhat/scylla.spec.mustache index 629053b2a4..d79559f6f2 100644 --- a/dist/redhat/scylla.spec.mustache +++ b/dist/redhat/scylla.spec.mustache @@ -49,7 +49,7 @@ BuildRequires: pystache {{#centos7}} Requires: kernel >= 3.10.0-514 {{/centos7}} -Requires: {{product}}-conf systemd-libs hwloc curl util-linux mdadm xfsprogs file {{product}}-python3 +Requires: {{product}}-conf systemd-libs hwloc curl util-linux mdadm xfsprogs {{product}}-python3 Conflicts: abrt AutoReqProv: no From a3c1b9fcf3cb211085115e88a233cc137b0c6ef6 Mon Sep 17 00:00:00 2001 From: Takuya ASADA Date: Tue, 26 Feb 2019 04:26:43 +0000 Subject: [PATCH 3/4] dist/redhat: drop systemd-libs from dependency Since we switched to relocatable package, we don't need distribution native libraries, so the package is not needed anymore. --- dist/redhat/scylla.spec.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/redhat/scylla.spec.mustache b/dist/redhat/scylla.spec.mustache index d79559f6f2..3ab62c886c 100644 --- a/dist/redhat/scylla.spec.mustache +++ b/dist/redhat/scylla.spec.mustache @@ -49,7 +49,7 @@ BuildRequires: pystache {{#centos7}} Requires: kernel >= 3.10.0-514 {{/centos7}} -Requires: {{product}}-conf systemd-libs hwloc curl util-linux mdadm xfsprogs {{product}}-python3 +Requires: {{product}}-conf hwloc curl util-linux mdadm xfsprogs {{product}}-python3 Conflicts: abrt AutoReqProv: no From 75a785901934a6daeae65dba88e81d08dd6b1d44 Mon Sep 17 00:00:00 2001 From: Takuya ASADA Date: Wed, 27 Feb 2019 09:21:00 +0000 Subject: [PATCH 4/4] reloc: add relocatable CLI tools for scylla setup scripts To minimize dependencies of Scylla, add relocatable image of CLI tools required for scylla setup scripts. --- dist/common/scripts/node_health_check | 26 +++++++------------------- dist/common/scripts/scylla_util.py | 12 ++++++++---- dist/redhat/scylla.spec.mustache | 2 +- install-dependencies.sh | 7 +++++++ scripts/create-relocatable-package.py | 10 +++++++++- 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/dist/common/scripts/node_health_check b/dist/common/scripts/node_health_check index dd07f08fc4..5466c12376 100755 --- a/dist/common/scripts/node_health_check +++ b/dist/common/scripts/node_health_check @@ -17,6 +17,8 @@ # You should have received a copy of the GNU General Public License # along with Scylla. If not, see . +# When a CLI tool is not installed, use relocatable CLI tool provided by Scylla +export PATH=$PATH:/opt/scylladb/bin ##Variables## REPORT="./`hostname`-health-check-report.txt" @@ -143,24 +145,6 @@ else echo "--------------------------------------------------" fi - -#Install 'net-tools' pkg, to be used for netstat command# -echo "Installing 'net-tools' Package (for 'netstat' command)" -echo "--------------------------------------------------" - -if [ "$IS_FEDORA" == "0" ]; then - sudo yum install net-tools -y -q -fi - -if [ "$IS_DEBIAN" == "0" ]; then -# sudo apt-get update -qq - sudo apt-get install net-tools -y | grep already -fi - -if [ "$IS_GENTOO" == "0" ]; then - sudo emerge -1uq sys-apps/ethtool sys-apps/net-tools -fi - #Create dir structure to save output_files# echo "--------------------------------------------------" echo "Creating Output Files Directory" @@ -282,7 +266,11 @@ for i in `ls -I lo /sys/class/net/`; do echo "--$i"; cat /sys/class/net/$i/queue for i in `ls -I lo /sys/class/net/`; do echo "--$i"; cat /sys/class/net/$i/queues/rx-*/rps_flow_cnt; echo ""; done > $OUTPUT_PATH5/rfs-conf.txt ps -elf | grep irqbalance > $OUTPUT_PATH5/irqbalance-conf.txt sudo sysctl -a > $OUTPUT_PATH5/sysctl.txt 2>&1 -sudo iptables -L -v > $OUTPUT_PATH5/iptables.txt +if [ -e /usr/sbin/iptables ]; then + sudo iptables -L -v > $OUTPUT_PATH5/iptables.txt +else + touch $OUTPUT_PATH5/iptables.txt +fi netstat -an | grep tcp > $OUTPUT_PATH5/netstat-tcp.txt diff --git a/dist/common/scripts/scylla_util.py b/dist/common/scripts/scylla_util.py index 391c6db379..d129be5c38 100644 --- a/dist/common/scripts/scylla_util.py +++ b/dist/common/scripts/scylla_util.py @@ -263,15 +263,19 @@ class scylla_cpuinfo: return len(self._cpu_data["system"]) +# When a CLI tool is not installed, use relocatable CLI tool provided by Scylla +scylla_env = os.environ.copy() +scylla_env['PATH'] = scylla_env['PATH'] + ':/opt/scylladb/bin' + def run(cmd, shell=False, silent=False, exception=True): stdout = subprocess.DEVNULL if silent else None stderr = subprocess.DEVNULL if silent else None if not shell: cmd = shlex.split(cmd) if exception: - return subprocess.check_call(cmd, shell=shell, stdout=stdout, stderr=stderr) + return subprocess.check_call(cmd, shell=shell, stdout=stdout, stderr=stderr, env=scylla_env) else: - p = subprocess.Popen(cmd, shell=shell, stdout=stdout, stderr=stderr) + p = subprocess.Popen(cmd, shell=shell, stdout=stdout, stderr=stderr, env=scylla_env) return p.wait() @@ -279,9 +283,9 @@ def out(cmd, shell=False, exception=True): if not shell: cmd = shlex.split(cmd) if exception: - return subprocess.check_output(cmd, shell=shell).strip().decode('utf-8') + return subprocess.check_output(cmd, shell=shell, env=scylla_env).strip().decode('utf-8') else: - p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE) + p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, env=scylla_env) return p.communicate()[0].strip().decode('utf-8') diff --git a/dist/redhat/scylla.spec.mustache b/dist/redhat/scylla.spec.mustache index 3ab62c886c..2a233037d2 100644 --- a/dist/redhat/scylla.spec.mustache +++ b/dist/redhat/scylla.spec.mustache @@ -49,7 +49,7 @@ BuildRequires: pystache {{#centos7}} Requires: kernel >= 3.10.0-514 {{/centos7}} -Requires: {{product}}-conf hwloc curl util-linux mdadm xfsprogs {{product}}-python3 +Requires: {{product}}-conf curl mdadm xfsprogs {{product}}-python3 Conflicts: abrt AutoReqProv: no diff --git a/install-dependencies.sh b/install-dependencies.sh index 0919b9f65c..91d1c307c3 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -61,6 +61,13 @@ fedora_packages=( python3-magic dnf-utils pigz + net-tools + tar + gzip + gawk + util-linux + ethtool + hwloc ) centos_packages=( diff --git a/scripts/create-relocatable-package.py b/scripts/create-relocatable-package.py index 36601e7539..c9ddc723c2 100755 --- a/scripts/create-relocatable-package.py +++ b/scripts/create-relocatable-package.py @@ -60,7 +60,15 @@ ap.add_argument('--mode', dest='mode', default='release', args = ap.parse_args() executables = ['build/{}/scylla'.format(args.mode), - 'build/{}/iotune'.format(args.mode)] + 'build/{}/iotune'.format(args.mode), + '/usr/bin/lscpu', + '/usr/bin/gawk', + '/usr/bin/gzip', + '/usr/sbin/ifconfig', + '/usr/sbin/ethtool', + '/usr/bin/netstat', + '/usr/bin/hwloc-distrib', + '/usr/bin/hwloc-calc'] output = args.dest