diff --git a/.zuul.yaml b/.zuul.yaml index 02e3ac7ac..2c15219c6 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -4,7 +4,7 @@ jobs: - openstack-tox-pep8: voting: false + - openstack-tox-linters gate: jobs: - - noop - + - openstack-tox-linters diff --git a/base/cluster-resource-agents/centos/meta_patches/PATCH_ORDER b/base/cluster-resource-agents/centos/meta_patches/PATCH_ORDER index 884d90399..e4b0c166e 100644 --- a/base/cluster-resource-agents/centos/meta_patches/PATCH_ORDER +++ b/base/cluster-resource-agents/centos/meta_patches/PATCH_ORDER @@ -10,3 +10,4 @@ ipaddr2-if-down.patch spec-add-ipaddr2-ignore-lo-state.patch Disable-creation-of-the-debug-package.patch metapatch-for-arp_bg.patch +ipaddr2-avoid-failing-svc-if-down-meta.patch diff --git a/base/cluster-resource-agents/centos/meta_patches/ipaddr2-avoid-failing-svc-if-down-meta.patch b/base/cluster-resource-agents/centos/meta_patches/ipaddr2-avoid-failing-svc-if-down-meta.patch new file mode 100644 index 000000000..5f1c0b88b --- /dev/null +++ b/base/cluster-resource-agents/centos/meta_patches/ipaddr2-avoid-failing-svc-if-down-meta.patch @@ -0,0 +1,32 @@ +From a8fc00f7d84327284643f70638da2201327bdd10 Mon Sep 17 00:00:00 2001 +From: Bin Qian +Date: Wed, 29 Aug 2018 11:22:32 -0400 +Subject: [PATCH 1/1] ipaddr2 avoid failing when svc i/f down + +--- + SPECS/resource-agents.spec | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/SPECS/resource-agents.spec b/SPECS/resource-agents.spec +index aa06c49..0eca09f 100644 +--- a/SPECS/resource-agents.spec ++++ b/SPECS/resource-agents.spec +@@ -257,6 +257,7 @@ Patch1118: ipaddr2_if_down.patch + Patch1119: ipaddr2_ignore_lo_if_state.patch + Patch1120: Modify-error-code-of-bz1454699-fix-to-prevent-inactive-controller-reboot-loop.patch + Patch1121: Re-enable-background-execution-of-arp-commands.patch ++Patch1122: ipaddr2-avoid-failing-svc-if-down.patch + + Obsoletes: heartbeat-resources <= %{version} + Provides: heartbeat-resources = %{version} +@@ -568,6 +569,7 @@ exit 1 + %patch1119 -p1 + %patch1120 -p1 + %patch1121 -p1 ++%patch1122 -p1 + + %build + if [ ! -f configure ]; then +-- +1.8.3.1 + diff --git a/base/cluster-resource-agents/centos/patches/ipaddr2-avoid-failing-svc-if-down.patch b/base/cluster-resource-agents/centos/patches/ipaddr2-avoid-failing-svc-if-down.patch new file mode 100644 index 000000000..fcc75030f --- /dev/null +++ b/base/cluster-resource-agents/centos/patches/ipaddr2-avoid-failing-svc-if-down.patch @@ -0,0 +1,61 @@ +From c3448b1536d50291dc5ca49dce5957c39403cc82 Mon Sep 17 00:00:00 2001 +From: Bin Qian +Date: Wed, 29 Aug 2018 11:00:22 -0400 +Subject: [PATCH 1/1] avoid failing service when I/F is down + +--- + heartbeat/IPaddr2 | 24 ++++++------------------ + 1 file changed, 6 insertions(+), 18 deletions(-) + +diff --git a/heartbeat/IPaddr2 b/heartbeat/IPaddr2 +index 86009b9..2da5c5e 100755 +--- a/heartbeat/IPaddr2 ++++ b/heartbeat/IPaddr2 +@@ -885,12 +885,8 @@ ip_start() { + then + exit $OCF_SUCCESS + else +- if [ "$OCF_RESKEY_dc" = "yes" ]; then +- ocf_log info "NIC $NIC is DOWN..." +- exit $OCF_SUCCESS +- else +- exit $OCF_ERR_GENERIC +- fi ++ ocf_log info "NIC $NIC is DOWN..." ++ exit $OCF_SUCCESS + fi + fi + +@@ -954,12 +950,8 @@ ip_start() { + then + exit $OCF_SUCCESS + else +- if [ "$OCF_RESKEY_dc" = "yes" ]; then +- ocf_log info "NIC $NIC is DOWN" +- exit $OCF_SUCCESS +- else +- exit $OCF_ERR_GENERIC +- fi ++ ocf_log info "NIC $NIC is DOWN" ++ exit $OCF_SUCCESS + fi + } + +@@ -1040,12 +1032,8 @@ ip_monitor() { + then + return $OCF_SUCCESS + else +- if [ "$OCF_RESKEY_dc" = "yes" ]; then +- ocf_log info "NIC $NIC is DOWN" +- return $OCF_SUCCESS +- else +- return $OCF_NOT_RUNNING +- fi ++ ocf_log info "NIC $NIC is DOWN" ++ return $OCF_SUCCESS + fi + ;; + partial|no|partial2) +-- +1.8.3.1 + diff --git a/base/haproxy/haproxy/haproxy.sh b/base/haproxy/haproxy/haproxy.sh index 560480e40..8791f9a1a 100755 --- a/base/haproxy/haproxy/haproxy.sh +++ b/base/haproxy/haproxy/haproxy.sh @@ -7,7 +7,7 @@ # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: HA-Proxy TCP/HTTP reverse proxy -# Description: HA-Proxy is a TCP/HTTP reverse proxy +# Description: HA-Proxy is a TCP/HTTP reverse proxy ### END INIT INFO PATH=/sbin:/bin:/usr/sbin:/usr/bin @@ -22,12 +22,12 @@ RETVAL=0 # This is only needed till TPM In-Kernel # ResourceMgr comes in remove_TPM_transients () { - _HANDLES=`find $TPM_DATA_DIR -type f -name "hp*.bin" -printf "%f "` - for handle in $_HANDLES; do + _HANDLES=`find $TPM_DATA_DIR -type f -name "hp*.bin" -printf "%f "` + for handle in $_HANDLES; do handle_addr=`echo $handle | sed 's/hp\([0-9]*\)\.bin/\1/g'` tss2_flushcontext -ha $handle_addr &> /dev/null - done - rm -f $TPM_DATA_DIR/* + done + rm -f $TPM_DATA_DIR/* } start() { @@ -59,10 +59,12 @@ start() { } stop() { - if [ ! -e $PIDFILE ]; then return; fi - + if [ ! -e $PIDFILE ]; then + return + fi + echo -n "Stopping $DESC..." - + start-stop-daemon --stop --quiet --retry 3 --oknodo --pidfile $PIDFILE -x "$DAEMON" if [ -n "`pidof $DAEMON`" ] ; then pkill -KILL -f $DAEMON @@ -73,8 +75,7 @@ stop() { remove_TPM_transients } -status() -{ +status() { pid=`cat $PIDFILE 2>/dev/null` if [ -n "$pid" ]; then if ps -p $pid &>/dev/null ; then @@ -90,7 +91,7 @@ status() } check() { - /usr/sbin/$NAME -c -q -V -f /etc/$NAME/$NAME.cfg + /usr/sbin/$NAME -c -q -V -f /etc/$NAME/$NAME.cfg } # See how we were called. @@ -116,5 +117,5 @@ case "$1" in RETVAL=1 ;; esac - + exit $RETVAL diff --git a/base/initscripts/centos/files/mountnfs.sh b/base/initscripts/centos/files/mountnfs.sh index acc07797b..adeea3d00 100755 --- a/base/initscripts/centos/files/mountnfs.sh +++ b/base/initscripts/centos/files/mountnfs.sh @@ -15,86 +15,83 @@ if [ "$1" = "stop" ]; then fi # -# Run in a subshell because of I/O redirection. +# Run in a subshell because of I/O redirection. # test -f /etc/fstab && ( # -# Read through fstab line by line. If it is NFS, set the flag -# for mounting NFS filesystems. If any NFS partition is found and it -# not mounted with the nolock option, we start the rpcbind. +# Read through fstab line by line. If it is NFS, set the flag +# for mounting NFS filesystems. If any NFS partition is found and it +# not mounted with the nolock option, we start the rpcbind. # rpcbind=no mount_nfs=no mount_smb=no mount_ncp=no mount_cifs=no -while read device mountpt fstype options -do - case "$device" in - ""|\#*) - continue - ;; - esac +while read device mountpt fstype options; do + case "$device" in + ""|\#*) + continue + ;; + esac - case "$options" in - *noauto*) - continue - ;; - esac + case "$options" in + *noauto*) + continue + ;; + esac - if test "$fstype" = nfs - then - mount_nfs=yes - case "$options" in - *nolock*) - ;; - *) - rpcbind=yes - ;; - esac - fi - if test "$fstype" = smbfs - then - mount_smb=yes - fi - if test "$fstype" = ncpfs - then - mount_ncp=yes - fi - if test "$fstype" = cifs - then - mount_cifs=yes - fi + if test "$fstype" = nfs + then + mount_nfs=yes + case "$options" in + *nolock*) + ;; + *) + rpcbind=yes + ;; + esac + fi + if test "$fstype" = smbfs + then + mount_smb=yes + fi + if test "$fstype" = ncpfs + then + mount_ncp=yes + fi + if test "$fstype" = cifs + then + mount_cifs=yes + fi done exec 0>&1 -if test "$rpcbind" = yes -then +if test "$rpcbind" = yes; then # WRL: Centos precheck: Dont start rpcbind in this init script. # It is started by a systemd service file. if test "/etc/centos-release" = no then if test -x /usr/sbin/rpcbind then - service rpcbind status > /dev/null - if [ $? != 0 ]; then - echo -n "Starting rpcbind..." - start-stop-daemon --start --quiet --exec /usr/sbin/rpcbind - sleep 2 + service rpcbind status > /dev/null + if [ $? != 0 ]; then + echo -n "Starting rpcbind..." + start-stop-daemon --start --quiet --exec /usr/sbin/rpcbind + sleep 2 fi fi fi fi -if test "$mount_nfs" = yes || test "$mount_smb" = yes || test "$mount_ncp" = yes || test "$mount_cifs" = yes -then - echo "Mounting remote filesystems..." - test "$mount_nfs" = yes && mount -a -t nfs - test "$mount_smb" = yes && mount -a -t smbfs - test "$mount_ncp" = yes && mount -a -t ncpfs - test "$mount_cifs" = yes && mount -a -t cifs +if test "$mount_nfs" = yes || test "$mount_smb" = yes || test "$mount_ncp" = yes || test "$mount_cifs" = yes; then + echo "Mounting remote filesystems..." + test "$mount_nfs" = yes && mount -a -t nfs + test "$mount_smb" = yes && mount -a -t smbfs + test "$mount_ncp" = yes && mount -a -t ncpfs + test "$mount_cifs" = yes && mount -a -t cifs fi ) < /etc/fstab diff --git a/base/memcached/centos/build_srpm.data b/base/memcached/centos/build_srpm.data deleted file mode 100644 index 8aeb55368..000000000 --- a/base/memcached/centos/build_srpm.data +++ /dev/null @@ -1 +0,0 @@ -TIS_PATCH_VER=1 diff --git a/base/memcached/centos/meta_patches/0001-Update-package-versioning-for-TIS-format.patch b/base/memcached/centos/meta_patches/0001-Update-package-versioning-for-TIS-format.patch deleted file mode 100644 index 5625e1b28..000000000 --- a/base/memcached/centos/meta_patches/0001-Update-package-versioning-for-TIS-format.patch +++ /dev/null @@ -1,25 +0,0 @@ -From de355606dea0404c4ae92bad5ce00b841697c698 Mon Sep 17 00:00:00 2001 -From: Jack Ding -Date: Tue, 8 May 2018 14:29:14 -0400 -Subject: [PATCH] Update package versioning for TIS format - ---- - SPECS/memcached.spec | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/SPECS/memcached.spec b/SPECS/memcached.spec -index 6008493..c8575c8 100644 ---- a/SPECS/memcached.spec -+++ b/SPECS/memcached.spec -@@ -4,7 +4,7 @@ - - Name: memcached - Version: 1.4.39 --Release: 1%{?dist} -+Release: 1.el7%{?_tis_dist}.%{tis_patch_ver} - Epoch: 0 - Summary: High Performance, Distributed Memory Object Cache - --- -1.8.3.1 - diff --git a/base/memcached/centos/meta_patches/0002-always-restart-memcached-service.patch b/base/memcached/centos/meta_patches/0002-always-restart-memcached-service.patch deleted file mode 100644 index 89cf9c2ee..000000000 --- a/base/memcached/centos/meta_patches/0002-always-restart-memcached-service.patch +++ /dev/null @@ -1,32 +0,0 @@ -From f321c8a8b800a7c2ca9394d3c76bec72b98c0d77 Mon Sep 17 00:00:00 2001 -From: Jack Ding -Date: Fri, 11 May 2018 15:38:56 -0400 -Subject: [PATCH] always restart memcached service - ---- - SPECS/memcached.spec | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/SPECS/memcached.spec b/SPECS/memcached.spec -index c8575c8..f389035 100644 ---- a/SPECS/memcached.spec -+++ b/SPECS/memcached.spec -@@ -16,6 +16,7 @@ Source1: memcached.sysconfig - - # https://github.com/memcached/memcached/issues/218 - Patch1: 0001-systemd-fix-upstream-provided-service.patch -+Patch2: 0002-always-restart-memcached-service.patch - - BuildRequires: libevent-devel systemd-units - BuildRequires: perl-generators -@@ -44,6 +45,7 @@ access to the memcached binary include files. - %prep - %setup -q - %patch1 -p1 -b .unit -+%patch2 -p1 - - %build - # compile with full RELRO --- -1.8.3.1 - diff --git a/base/memcached/centos/meta_patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch b/base/memcached/centos/meta_patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch deleted file mode 100644 index 93beb0546..000000000 --- a/base/memcached/centos/meta_patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch +++ /dev/null @@ -1,33 +0,0 @@ -From bb6fd3da3ace960eb587e7ff01d5816ea2baaa54 Mon Sep 17 00:00:00 2001 -From: Jack Ding -Date: Sun, 13 May 2018 18:22:15 -0400 -Subject: [PATCH] Add dependencies and comment out incompatible service - parameters - ---- - SPECS/memcached.spec | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/SPECS/memcached.spec b/SPECS/memcached.spec -index f389035..86653a1 100644 ---- a/SPECS/memcached.spec -+++ b/SPECS/memcached.spec -@@ -17,6 +17,7 @@ Source1: memcached.sysconfig - # https://github.com/memcached/memcached/issues/218 - Patch1: 0001-systemd-fix-upstream-provided-service.patch - Patch2: 0002-always-restart-memcached-service.patch -+Patch3: 0003-Add-dependencies-and-comment-out-incompatible-servic.patch - - BuildRequires: libevent-devel systemd-units - BuildRequires: perl-generators -@@ -46,6 +47,7 @@ access to the memcached binary include files. - %setup -q - %patch1 -p1 -b .unit - %patch2 -p1 -+%patch3 -p1 - - %build - # compile with full RELRO --- -1.8.3.1 - diff --git a/base/memcached/centos/meta_patches/PATCH_ORDER b/base/memcached/centos/meta_patches/PATCH_ORDER deleted file mode 100644 index 163c84245..000000000 --- a/base/memcached/centos/meta_patches/PATCH_ORDER +++ /dev/null @@ -1,3 +0,0 @@ -0001-Update-package-versioning-for-TIS-format.patch -0002-always-restart-memcached-service.patch -0003-Add-dependencies-and-comment-out-incompatible-servic.patch diff --git a/base/memcached/centos/patches/0002-always-restart-memcached-service.patch b/base/memcached/centos/patches/0002-always-restart-memcached-service.patch deleted file mode 100644 index 5f899f13e..000000000 --- a/base/memcached/centos/patches/0002-always-restart-memcached-service.patch +++ /dev/null @@ -1,26 +0,0 @@ -From bb7b75184f7037e6d8d844874ae248fce1d06736 Mon Sep 17 00:00:00 2001 -From: Jack Ding -Date: Fri, 11 May 2018 15:24:28 -0400 -Subject: [PATCH] Always restart memcached service - ---- - scripts/memcached.service | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/scripts/memcached.service b/scripts/memcached.service -index 1bb9d33..8e58485 100644 ---- a/scripts/memcached.service -+++ b/scripts/memcached.service -@@ -71,5 +71,9 @@ RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX - # Takes away the ability to create or manage any kind of namespace - RestrictNamespaces=true - -+# WRS -+Restart=always -+RestartSec=0 -+ - [Install] - WantedBy=multi-user.target --- -1.8.3.1 - diff --git a/base/memcached/centos/patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch b/base/memcached/centos/patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch deleted file mode 100644 index a2ef1db58..000000000 --- a/base/memcached/centos/patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch +++ /dev/null @@ -1,67 +0,0 @@ -From 1d9f43c5ecb20fe0a2a4abe9b94abd0d389edb40 Mon Sep 17 00:00:00 2001 -From: Jack Ding -Date: Mon, 14 May 2018 22:44:32 -0400 -Subject: [PATCH 2/2] Add dependencies and comment out incompatible service - parameters - ---- - scripts/memcached.service | 14 +++++++------- - 1 file changed, 7 insertions(+), 7 deletions(-) - -diff --git a/scripts/memcached.service b/scripts/memcached.service -index 8e58485..021b8b4 100644 ---- a/scripts/memcached.service -+++ b/scripts/memcached.service -@@ -12,7 +12,7 @@ - [Unit] - Description=memcached daemon - Before=httpd.service --After=network.target -+After=network-online.target - - [Service] - EnvironmentFile=/etc/sysconfig/memcached -@@ -46,34 +46,34 @@ LimitNOFILE=16384 - # Explicit module loading will be denied. This allows to turn off module load and unload - # operations on modular kernels. It is recommended to turn this on for most services that - # do not need special file systems or extra kernel modules to work. --ProtectKernelModules=true -+#ProtectKernelModules=true - - # Kernel variables accessible through /proc/sys, /sys, /proc/sysrq-trigger, /proc/latency_stats, - # /proc/acpi, /proc/timer_stats, /proc/fs and /proc/irq will be made read-only to all processes - # of the unit. Usually, tunable kernel variables should only be written at boot-time, with the - # sysctl.d(5) mechanism. Almost no services need to write to these at runtime; it is hence - # recommended to turn this on for most services. --ProtectKernelTunables=true -+#ProtectKernelTunables=true - - # The Linux Control Groups (cgroups(7)) hierarchies accessible through /sys/fs/cgroup will be - # made read-only to all processes of the unit. Except for container managers no services should - # require write access to the control groups hierarchies; it is hence recommended to turn this on - # for most services --ProtectControlGroups=true -+#ProtectControlGroups=true - - # Any attempts to enable realtime scheduling in a process of the unit are refused. --RestrictRealtime=true -+#RestrictRealtime=true - - # Restricts the set of socket address families accessible to the processes of this unit. - # Protects against vulnerabilities such as CVE-2016-8655 - RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX - - # Takes away the ability to create or manage any kind of namespace --RestrictNamespaces=true -+#RestrictNamespaces=true - - # WRS - Restart=always --RestartSec=0 -+RestartSec=10 - - [Install] - WantedBy=multi-user.target --- -1.8.3.1 - diff --git a/base/memcached/centos/srpm_path b/base/memcached/centos/srpm_path deleted file mode 100644 index dc4b6de47..000000000 --- a/base/memcached/centos/srpm_path +++ /dev/null @@ -1 +0,0 @@ -mirror:Source/memcached-1.4.39-1.el7.src.rpm diff --git a/base/novnc/centos/build_srpm.data b/base/novnc/centos/build_srpm.data deleted file mode 100644 index 8aeb55368..000000000 --- a/base/novnc/centos/build_srpm.data +++ /dev/null @@ -1 +0,0 @@ -TIS_PATCH_VER=1 diff --git a/base/novnc/centos/meta_patches/0001-Update-package-0.6.2-versioning-for-TIS-format.patch b/base/novnc/centos/meta_patches/0001-Update-package-0.6.2-versioning-for-TIS-format.patch deleted file mode 100644 index 062bfe822..000000000 --- a/base/novnc/centos/meta_patches/0001-Update-package-0.6.2-versioning-for-TIS-format.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/SPECS/novnc.spec b/SPECS/novnc.spec -index a43f435..640bf1f 100644 ---- a/SPECS/novnc.spec -+++ b/SPECS/novnc.spec -@@ -1,6 +1,6 @@ - Name: novnc - Version: 0.6.2 --Release: 1%{?dist} -+Release: 1.el7%{?_tis_dist}.%{tis_patch_ver} - Summary: VNC client using HTML5 (Web Sockets, Canvas) with encryption support - Requires: python-websockify - diff --git a/base/novnc/centos/meta_patches/PATCH_ORDER b/base/novnc/centos/meta_patches/PATCH_ORDER deleted file mode 100644 index 0eca01bae..000000000 --- a/base/novnc/centos/meta_patches/PATCH_ORDER +++ /dev/null @@ -1 +0,0 @@ -0001-Update-package-0.6.2-versioning-for-TIS-format.patch diff --git a/base/novnc/centos/srpm_path b/base/novnc/centos/srpm_path deleted file mode 100644 index d59f7598d..000000000 --- a/base/novnc/centos/srpm_path +++ /dev/null @@ -1 +0,0 @@ -mirror:Source/novnc-0.6.2-1.el7.src.rpm diff --git a/centos_iso_image.inc b/centos_iso_image.inc index 3ebf82576..328facc58 100644 --- a/centos_iso_image.inc +++ b/centos_iso_image.inc @@ -104,6 +104,7 @@ collector # platform-util platform-util platform-util-noncontroller +platform-util-controller # monitor-tools monitor-tools diff --git a/centos_pkg_dirs b/centos_pkg_dirs index d04e605a4..d3ab43e6c 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -32,7 +32,6 @@ base/shadow-utils security/shim-unsigned security/shim-signed logging/syslog-ng -base/novnc base/sanlock base/sudo virt/cloud-init @@ -88,13 +87,12 @@ python/python-requests base/systemd python/python-gunicorn config/puppet-modules/openstack/puppet-oslo-11.3.0 -strorage-drivers/python-3parclient -strorage-drivers/python-lefthandclient +storage-drivers/python-3parclient +storage-drivers/python-lefthandclient security/tboot networking/mellanox/libibverbs kernel/kernel-modules/mlnx-ofa_kernel networking/mellanox/rdma-core -base/memcached config/puppet-modules/openstack/puppet-memcached-3.0.2 config/puppet-modules/openstack/puppet-horizon-9.5.0 config/puppet-modules/openstack/puppet-swift-11.3.0 diff --git a/database/mariadb/centos/files/clustercheck.sh b/database/mariadb/centos/files/clustercheck.sh index 782dbb6f1..ce16a81dc 100644 --- a/database/mariadb/centos/files/clustercheck.sh +++ b/database/mariadb/centos/files/clustercheck.sh @@ -43,8 +43,7 @@ fi WSREP_STATUS=$($MYSQL_CMDLINE -e "SHOW STATUS LIKE 'wsrep_local_state';" \ 2>${ERR_FILE} | tail -1 2>>${ERR_FILE}) -if [[ "${WSREP_STATUS}" == "4" ]] || [[ "${WSREP_STATUS}" == "2" && ${AVAILABLE_WHEN_DONOR} == 1 ]] -then +if [[ "${WSREP_STATUS}" == "4" ]] || [[ "${WSREP_STATUS}" == "2" && ${AVAILABLE_WHEN_DONOR} == 1 ]]; then # Check only when set to 0 to avoid latency in response. if [[ $AVAILABLE_WHEN_READONLY -eq 0 ]];then READ_ONLY=$($MYSQL_CMDLINE -e "SHOW GLOBAL VARIABLES LIKE 'read_only';" \ diff --git a/database/mariadb/centos/files/mysql-check-upgrade.sh b/database/mariadb/centos/files/mysql-check-upgrade.sh index 1bfd3bc6a..cd669bda8 100644 --- a/database/mariadb/centos/files/mysql-check-upgrade.sh +++ b/database/mariadb/centos/files/mysql-check-upgrade.sh @@ -27,9 +27,9 @@ if [ $version -ne $thisversion ] ; then cat <&2 The datadir located at $datadir needs to be upgraded using 'mysql_upgrade' tool. This can be done using the following steps: - 1. Back-up your data before with 'mysql_upgrade' - 2. Start the database daemon using 'service @DAEMON_NAME@ start' - 3. Run 'mysql_upgrade' with a database user that has sufficient privileges + 1. Back-up your data before with 'mysql_upgrade' + 2. Start the database daemon using 'service @DAEMON_NAME@ start' + 3. Run 'mysql_upgrade' with a database user that has sufficient privileges Read more about 'mysql_upgrade' usage at: https://mariadb.com/kb/en/mariadb/documentation/sql-commands/table-commands/mysql_upgrade/ diff --git a/database/mariadb/centos/files/mysql-prepare-db-dir.sh b/database/mariadb/centos/files/mysql-prepare-db-dir.sh index b47fa9b4a..52a42a067 100644 --- a/database/mariadb/centos/files/mysql-prepare-db-dir.sh +++ b/database/mariadb/centos/files/mysql-prepare-db-dir.sh @@ -27,30 +27,26 @@ should_initialize () # If two args given first is user, second is group # otherwise the arg is the systemd service file -if [ "$#" -eq 2 ] -then +if [ "$#" -eq 2 ]; then myuser="$1" mygroup="$2" else # Absorb configuration settings from the specified systemd service file, # or the default service if not specified SERVICE_NAME="$1" - if [ x"$SERVICE_NAME" = x ] - then + if [ x"$SERVICE_NAME" = x ]; then SERVICE_NAME=@DAEMON_NAME@.service fi myuser=`systemctl show -p User "${SERVICE_NAME}" | - sed 's/^User=//'` - if [ x"$myuser" = x ] - then + sed 's/^User=//'` + if [ x"$myuser" = x ]; then myuser=mysql fi mygroup=`systemctl show -p Group "${SERVICE_NAME}" | - sed 's/^Group=//'` - if [ x"$mygroup" = x ] - then + sed 's/^Group=//'` + if [ x"$mygroup" = x ]; then mygroup=mysql fi fi @@ -79,8 +75,7 @@ chmod 0640 "$errlogfile" if should_initialize "$datadir" ; then # First, make sure $datadir is there with correct permissions # (note: if it's not, and we're not root, this'll fail ...) - if [ ! -e "$datadir" -a ! -h "$datadir" ] - then + if [ ! -e "$datadir" -a ! -h "$datadir" ]; then mkdir -p "$datadir" || exit 1 fi chown "$myuser:$mygroup" "$datadir" diff --git a/database/mariadb/centos/files/mysql-scripts-common.sh b/database/mariadb/centos/files/mysql-scripts-common.sh index bf2888f2a..f77a8040b 100644 --- a/database/mariadb/centos/files/mysql-scripts-common.sh +++ b/database/mariadb/centos/files/mysql-scripts-common.sh @@ -11,18 +11,18 @@ export LC_ALL=C # We use my_print_defaults which prints all options from multiple files, # with the more specific ones later; hence take the last match. get_mysql_option(){ - if [ $# -ne 3 ] ; then - echo "get_mysql_option requires 3 arguments: section option default_value" - return - fi - sections="$1" - option_name="$2" - default_value="$3" - result=`@bindir@/my_print_defaults $sections | sed -n "s/^--${option_name}=//p" | tail -n 1` - if [ -z "$result" ]; then - # not found, use default - result="${default_value}" - fi + if [ $# -ne 3 ] ; then + echo "get_mysql_option requires 3 arguments: section option default_value" + return + fi + sections="$1" + option_name="$2" + default_value="$3" + result=`@bindir@/my_print_defaults $sections | sed -n "s/^--${option_name}=//p" | tail -n 1` + if [ -z "$result" ]; then + # not found, use default + result="${default_value}" + fi } # Defaults here had better match what mysqld_safe will default to diff --git a/database/mariadb/centos/files/mysql-wait-ready.sh b/database/mariadb/centos/files/mysql-wait-ready.sh index 2ed5fe1d3..6d95800aa 100644 --- a/database/mariadb/centos/files/mysql-wait-ready.sh +++ b/database/mariadb/centos/files/mysql-wait-ready.sh @@ -9,8 +9,8 @@ source "`dirname ${BASH_SOURCE[0]}`/mysql-scripts-common" # "After mysqld" won't be started until the database is really ready. if [ $# -ne 1 ] ; then - echo "You need to pass daemon pid as an argument for this script." - exit 20 + echo "You need to pass daemon pid as an argument for this script." + exit 20 fi # Service file passes us the daemon's PID (actually, mysqld_safe's PID) @@ -19,27 +19,27 @@ daemon_pid="$1" # Wait for the server to come up or for the mysqld process to disappear ret=0 while /bin/true; do - # Check process still exists - if ! [ -d "/proc/${daemon_pid}" ] ; then - ret=1 - break - fi - RESPONSE=`@bindir@/mysqladmin --no-defaults --socket="$socketfile" --user=UNKNOWN_MYSQL_USER ping 2>&1` - mret=$? - if [ $mret -eq 0 ] ; then - break - fi - # exit codes 1, 11 (EXIT_CANNOT_CONNECT_TO_SERVICE) are expected, - # anything else suggests a configuration error - if [ $mret -ne 1 -a $mret -ne 11 ]; then + # Check process still exists + if ! [ -d "/proc/${daemon_pid}" ] ; then + ret=1 + break + fi + RESPONSE=`@bindir@/mysqladmin --no-defaults --socket="$socketfile" --user=UNKNOWN_MYSQL_USER ping 2>&1` + mret=$? + if [ $mret -eq 0 ] ; then + break + fi + # exit codes 1, 11 (EXIT_CANNOT_CONNECT_TO_SERVICE) are expected, + # anything else suggests a configuration error + if [ $mret -ne 1 -a $mret -ne 11 ]; then echo "Cannot check for @NICE_PROJECT_NAME@ Daemon startup because of mysqladmin failure." >&2 - ret=$mret - break - fi - # "Access denied" also means the server is alive - echo "$RESPONSE" | grep -q "Access denied for user" && break + ret=$mret + break + fi + # "Access denied" also means the server is alive + echo "$RESPONSE" | grep -q "Access denied for user" && break - sleep 1 + sleep 1 done exit $ret diff --git a/database/mariadb/centos/files/mysql-wait-stop.sh b/database/mariadb/centos/files/mysql-wait-stop.sh index 62bde30ea..0af45fd51 100644 --- a/database/mariadb/centos/files/mysql-wait-stop.sh +++ b/database/mariadb/centos/files/mysql-wait-stop.sh @@ -13,24 +13,24 @@ TIMEOUT=60 SECONDS=0 if ! [ -f "$pidfile" ]; then - exit 0 + exit 0 fi MYSQLPID=`cat "$pidfile" 2>/dev/null` if [ -z "$MYSQLPID" ] ; then - exit 2 + exit 2 fi while /bin/true; do - # Check process still exists - if ! [ -d "/proc/${MYSQLPID}" ] ; then - break - fi - if [ $SECONDS -gt $TIMEOUT ] ; then - ret=3 - break - fi - sleep 1 + # Check process still exists + if ! [ -d "/proc/${MYSQLPID}" ] ; then + break + fi + if [ $SECONDS -gt $TIMEOUT ] ; then + ret=3 + break + fi + sleep 1 done exit $ret diff --git a/filesystem/nfscheck/files/nfscheck-init.sh b/filesystem/nfscheck/files/nfscheck-init.sh index ac2d86920..53b1fccea 100755 --- a/filesystem/nfscheck/files/nfscheck-init.sh +++ b/filesystem/nfscheck/files/nfscheck-init.sh @@ -22,7 +22,7 @@ DESC="nfscheck" DAEMON="/usr/bin/nfscheck" PIDFILE="/var/run/nfscheck.pid" -start() +start() { if [ -e $PIDFILE ]; then PIDDIR=/proc/$(cat $PIDFILE) @@ -47,7 +47,7 @@ start() fi } -stop() +stop() { echo -n "Stopping $DESC..." start-stop-daemon --stop --quiet --pidfile $PIDFILE diff --git a/filesystem/nfscheck/files/nfscheck.sh b/filesystem/nfscheck/files/nfscheck.sh index f58ab80ef..3b0679138 100644 --- a/filesystem/nfscheck/files/nfscheck.sh +++ b/filesystem/nfscheck/files/nfscheck.sh @@ -12,12 +12,10 @@ MOUNT=/opt/platform previous=1 delay=60 -while : -do +while : ; do # First, check that it's actually an NFS mount mount | grep -q $MOUNT - if [ $? -ne 0 ] - then + if [ $? -ne 0 ]; then logger -t NFSCHECK "$MOUNT is not mounted" previous=1 sleep $delay @@ -31,18 +29,15 @@ do # At this point, jobs will either report no jobs (empty) or Done, # unless the job is still running/hung rc=$(jobs) - if [[ -z "$rc" || $rc =~ "Done" ]] - then + if [[ -z "$rc" || $rc =~ "Done" ]]; then # NFS is successful - if [ $previous -ne 0 ] - then + if [ $previous -ne 0 ]; then logger -t NFSCHECK "NFS test of $MOUNT is ok" previous=0 fi else # Keep waiting until the job is done - while ! [[ -z "$rc" || $rc =~ "Done" ]] - do + while ! [[ -z "$rc" || $rc =~ "Done" ]]; do logger -t NFSCHECK "NFS test of $MOUNT is failed" previous=1 sleep $delay diff --git a/networking/lldpd/centos/files/i40e-lldp-configure.sh b/networking/lldpd/centos/files/i40e-lldp-configure.sh index 2796c4304..4a5e8a441 100644 --- a/networking/lldpd/centos/files/i40e-lldp-configure.sh +++ b/networking/lldpd/centos/files/i40e-lldp-configure.sh @@ -25,20 +25,17 @@ DEBUGFS_PATH=/sys/kernel/debug DEBUGFS_I40_DEVICES_PATH=$DEBUGFS_PATH/i40e LLDP_COMMAND=lldp -function log() -{ +function log { local MSG="${PROGNAME}: $1" - logger -p notice "${MSG}" + logger -p notice "${MSG}" } -function err() -{ +function err { local MSG="${PROGNAME}: $1" logger -p error "${MSG}" } -function configure_device() -{ +function configure_device { local DEVICE=$1 local ACTION=$2 local DEVICE_PATH=${DEBUGFS_I40_DEVICES}/${DEVICE} @@ -59,30 +56,29 @@ function configure_device() return ${RET} } -function is_debugfs_mounted() { - if grep -qs "${DEBUGFS_PATH}" /proc/mounts; then - return 0 - fi - return 1 +function is_debugfs_mounted { + if grep -qs "${DEBUGFS_PATH}" /proc/mounts; then + return 0 + fi + return 1 } -function mount_debugfs() { +function mount_debugfs { mount -t debugfs none ${DEBUGFS_PATH} } -function unmount_debugfs() { +function unmount_debugfs { umount ${DEBUGFS_PATH} } -function scan_devices() -{ +function scan_devices { local ACTION=$1 local DEBUGFS_MOUNTED="false" local DEVICES=${DEBUGFS_I40_DEVICES_PATH}/* if is_debugfs_mounted; then DEBUGFS_MOUNTED="true" - fi + fi if [ ${DEBUGFS_MOUNTED} = "false" ]; then mount_debugfs @@ -111,38 +107,35 @@ function scan_devices() return 0 } -function start() -{ +function start { scan_devices start return $? } -function stop() -{ +function stop { scan_devices stop - return $? + return $? } -function status() -{ +function status { return 0 } case "$1" in - start) + start) start ;; - stop) + stop) stop ;; - restart) + restart) stop start ;; - status) + status) status ;; - *) + *) echo "Usage: $0 {start|stop|restart|status}" exit 1 esac diff --git a/networking/mellanox/mlx4-config/files/mlx4-configure.sh b/networking/mellanox/mlx4-config/files/mlx4-configure.sh index 0953f0775..959024e09 100644 --- a/networking/mellanox/mlx4-config/files/mlx4-configure.sh +++ b/networking/mellanox/mlx4-config/files/mlx4-configure.sh @@ -15,17 +15,15 @@ DEBUG=${DEBUG:-0} # all files beginning in "mlx4_port" and ending in one or more digits. shopt -s extglob -function log() -{ +function log { local MSG="${PROGNAME}: $1" if [ ${DEBUG} -ne 0 ]; then - echo "${MSG}" + echo "${MSG}" fi echo "${MSG}" >> /var/log/mlx4-configure.log } -function configure_device() -{ +function configure_device { local DEVICE=$1 local DEVICE_PATH=${SYSFS_PCI_DEVICES}/${DEVICE} @@ -58,8 +56,7 @@ function configure_device() } -function scan_devices() -{ +function scan_devices { local DEVICES=$(ls -1 ${SYSFS_PCI_DEVICES}) for DEVICE in ${DEVICES}; do @@ -83,37 +80,34 @@ function scan_devices() } -function start() -{ +function start { scan_devices return $? } -function stop() -{ +function stop { return 0 } -function status() -{ +function status { return 0 } case "$1" in - start) + start) start ;; - stop) + stop) stop ;; - restart) + restart) stop start ;; - status) + status) status ;; - *) + *) echo "Usage: $0 {start|stop|restart|status}" exit 1 esac diff --git a/networking/mellanox/mlx4-config/files/mlx4_core_config.sh b/networking/mellanox/mlx4-config/files/mlx4_core_config.sh index 94816bff0..b1c0652d8 100644 --- a/networking/mellanox/mlx4-config/files/mlx4_core_config.sh +++ b/networking/mellanox/mlx4-config/files/mlx4_core_config.sh @@ -1,7 +1,7 @@ #!/bin/bash ################################################################################ # Copyright (c) 2015-2016 Wind River Systems, Inc. -# +# # SPDX-License-Identifier: Apache-2.0 # ################################################################################ diff --git a/networking/mellanox/mlx4-config/files/mlx4_core_goenabled.sh b/networking/mellanox/mlx4-config/files/mlx4_core_goenabled.sh index c1a3592d7..189cf3214 100644 --- a/networking/mellanox/mlx4-config/files/mlx4_core_goenabled.sh +++ b/networking/mellanox/mlx4-config/files/mlx4_core_goenabled.sh @@ -12,13 +12,11 @@ NAME=$(basename $0) OPTIONS_CHANGED_FLAG=/var/run/.mlx4_cx3_reboot_required COMPUTE_CONFIG_COMPLETE=/var/run/.compute_config_complete -function LOG() -{ +function LOG { logger "$NAME: $*" } -if [ -f $OPTIONS_CHANGED_FLAG ] && [ -f $COMPUTE_CONFIG_COMPLETE ] -then +if [ -f $OPTIONS_CHANGED_FLAG ] && [ -f $COMPUTE_CONFIG_COMPLETE ]; then LOG "mlx4_core options has been changed. Failing goenabled check." exit 1 fi diff --git a/strorage-drivers/python-3parclient/centos/build_srpm.data b/storage-drivers/python-3parclient/centos/build_srpm.data similarity index 100% rename from strorage-drivers/python-3parclient/centos/build_srpm.data rename to storage-drivers/python-3parclient/centos/build_srpm.data diff --git a/strorage-drivers/python-3parclient/centos/python-3parclient.spec b/storage-drivers/python-3parclient/centos/python-3parclient.spec similarity index 100% rename from strorage-drivers/python-3parclient/centos/python-3parclient.spec rename to storage-drivers/python-3parclient/centos/python-3parclient.spec diff --git a/strorage-drivers/python-lefthandclient/centos/build_srpm.data b/storage-drivers/python-lefthandclient/centos/build_srpm.data similarity index 100% rename from strorage-drivers/python-lefthandclient/centos/build_srpm.data rename to storage-drivers/python-lefthandclient/centos/build_srpm.data diff --git a/strorage-drivers/python-lefthandclient/centos/python-lefthandclient.spec b/storage-drivers/python-lefthandclient/centos/python-lefthandclient.spec similarity index 100% rename from strorage-drivers/python-lefthandclient/centos/python-lefthandclient.spec rename to storage-drivers/python-lefthandclient/centos/python-lefthandclient.spec diff --git a/tools/collector/centos/build_srpm.data b/tools/collector/centos/build_srpm.data index 69bf0121c..66f0fdbd8 100644 --- a/tools/collector/centos/build_srpm.data +++ b/tools/collector/centos/build_srpm.data @@ -1,2 +1,2 @@ SRC_DIR="scripts" -TIS_PATCH_VER=24 +TIS_PATCH_VER=25 diff --git a/tools/collector/scripts/collect_ceph.sh b/tools/collector/scripts/collect_ceph.sh index 0898d8db8..1a5863ed1 100755 --- a/tools/collector/scripts/collect_ceph.sh +++ b/tools/collector/scripts/collect_ceph.sh @@ -14,8 +14,7 @@ SERVICE="ceph" LOGFILE="${extradir}/ceph.info" echo "${hostname}: Ceph Info .........: ${LOGFILE}" -function is_service_active() -{ +function is_service_active { active=`sm-query service management-ip | grep "enabled-active"` if [ -z "$active" ] ; then return 0 @@ -24,8 +23,7 @@ function is_service_active() fi } -function exit_if_timeout() -{ +function exit_if_timeout { if [ "$?" = "124" ] ; then echo "Exiting due to ceph command timeout" >> ${LOGFILE} exit 0 diff --git a/tools/collector/scripts/collect_ima.sh b/tools/collector/scripts/collect_ima.sh index dd35bfa63..14c751e42 100755 --- a/tools/collector/scripts/collect_ima.sh +++ b/tools/collector/scripts/collect_ima.sh @@ -10,8 +10,7 @@ source /usr/local/sbin/collect_parms source /usr/local/sbin/collect_utils -function is_extended_profile() -{ +function is_extended_profile { if [ ! -n "${security_profile}" ] || [ "${security_profile}" != "extended" ]; then return 0 else diff --git a/tools/collector/scripts/collect_nfv_vim.sh b/tools/collector/scripts/collect_nfv_vim.sh index 4ccd309de..c5ccbc7fa 100644 --- a/tools/collector/scripts/collect_nfv_vim.sh +++ b/tools/collector/scripts/collect_nfv_vim.sh @@ -12,8 +12,7 @@ source /usr/local/sbin/collect_utils LOGFILE="${extradir}/nfv-vim.info" echo "${hostname}: NFV-Vim Info ......: ${LOGFILE}" -function is_service_active() -{ +function is_service_active { active=`sm-query service vim | grep "enabled-active"` if [ -z "$active" ] ; then return 0 @@ -36,7 +35,7 @@ if [ "$nodetype" = "controller" ] ; then DATABASE_DIR=$(awk -F "=" '/database_dir/ {print $2}' /etc/nfv/vim/config.ini) SQLITE_DUMP="/usr/bin/sqlite3 ${DATABASE_DIR}/vim_db_v1 .dump" - + delimiter ${LOGFILE} "dump database" timeout 30 ${SQLITE_DUMP} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} fi diff --git a/tools/collector/scripts/collect_openstack.sh b/tools/collector/scripts/collect_openstack.sh index 743dd5d56..42d7d24d8 100755 --- a/tools/collector/scripts/collect_openstack.sh +++ b/tools/collector/scripts/collect_openstack.sh @@ -10,8 +10,7 @@ source /usr/local/sbin/collect_parms source /usr/local/sbin/collect_utils -function is_service_active() -{ +function is_service_active { active=`sm-query service rabbit-fs | grep "enabled-active"` if [ -z "$active" ] ; then return 0 @@ -51,7 +50,7 @@ num_exchanges=$(rabbitmqctl list_exchanges | wc -l); ((num_exchanges-=2)) num_connections=$(rabbitmqctl list_connections | wc -l); ((num_connections-=2)) num_channels=$(rabbitmqctl list_channels | wc -l); ((num_channels-=2)) arr=($(rabbitmqctl list_queues messages consumers memory | \ - awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) + awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]} printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" "queues" "bindings" "exchanges" "connections" "channels" "messages" "consumers" "memory" >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" $num_queues $num_bindings $num_exchanges $num_connections $num_channels $messages $consumers $memory >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} diff --git a/tools/collector/scripts/collect_psqldb.sh b/tools/collector/scripts/collect_psqldb.sh index 410aefa5c..d223b1b7b 100755 --- a/tools/collector/scripts/collect_psqldb.sh +++ b/tools/collector/scripts/collect_psqldb.sh @@ -19,8 +19,7 @@ DB_DIR="${extradir}/database" LOGFILE="${extradir}/database.info" echo "${hostname}: Database Info .....: ${LOGFILE}" -function is_service_active() -{ +function is_service_active { active=`sm-query service postgres | grep "enabled-active"` if [ -z "$active" ] ; then return 0 @@ -34,11 +33,9 @@ function is_service_active() ############################################################################### mkdir -p ${DB_DIR} -function log_database() -{ +function log_database { db_list=( $(${PSQL_CMD} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) - for db in "${db_list[@]}" - do + for db in "${db_list[@]}"; do echo "postgres database: ${db}" ${PSQL_CMD} -d ${db} -c " SELECT @@ -75,15 +72,13 @@ function log_database() DB_EXT=db.sql.txt -function database_dump() -{ +function database_dump { mkdir -p ${DB_DIR} db_list=( $(${PSQL_CMD} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) - for DB in "${db_list[@]}" - do + for DB in "${db_list[@]}"; do if [ "$DB" != "keystone" -a "$DB" != "ceilometer" ] ; then echo "${hostname}: Dumping Database ..: ${DB_DIR}/$DB.$DB_EXT" - (cd ${DB_DIR} ; sudo -u postgres pg_dump $DB > $DB.$DB_EXT) + (cd ${DB_DIR} ; sudo -u postgres pg_dump $DB > $DB.$DB_EXT) fi done } @@ -107,7 +102,7 @@ if [ "$nodetype" = "controller" ] ; then pg_size_pretty(pg_database_size(pg_database.datname)) FROM pg_database ORDER BY pg_database_size DESC; - " >> ${LOGFILE} + " >> ${LOGFILE} # Number of postgres connections delimiter ${LOGFILE} "ps -C postgres -o cmd=" diff --git a/tools/collector/scripts/collect_sysinv.sh b/tools/collector/scripts/collect_sysinv.sh index 37ba26af5..3f27a2886 100755 --- a/tools/collector/scripts/collect_sysinv.sh +++ b/tools/collector/scripts/collect_sysinv.sh @@ -14,8 +14,7 @@ SERVICE="inventory" LOGFILE="${extradir}/${SERVICE}.info" RPMLOG="${extradir}/rpm.info" -function is_service_active() -{ +function is_service_active { active=`sm-query service management-ip | grep "enabled-active"` if [ -z "$active" ] ; then return 0 diff --git a/tools/collector/scripts/collect_tc.sh b/tools/collector/scripts/collect_tc.sh index 160b970e6..737461a88 100755 --- a/tools/collector/scripts/collect_tc.sh +++ b/tools/collector/scripts/collect_tc.sh @@ -27,8 +27,8 @@ fi delimiter ${LOGFILE} "ip link" ip link >> ${LOGFILE} -for i in $(ip link | grep mtu | grep eth |awk '{print $2}' | sed 's#:##g'); -do +for i in $(ip link | grep mtu | grep eth |awk '{print $2}' | sed 's#:##g'); do + delimiter ${LOGFILE} "ethtool ${i}" ethtool ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} @@ -55,8 +55,8 @@ fi delimiter ${LOGFILE} "tc qdisc show" tc qdisc show >> ${LOGFILE} -for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); -do +for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); do + delimiter ${LOGFILE} "tc class show dev ${i}" tc class show dev ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} @@ -70,8 +70,8 @@ done delimiter ${LOGFILE} "tc -s qdisc show" tc -s qdisc show >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} -for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); -do +for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); do + delimiter ${LOGFILE} "tc -s class show dev ${i}" tc -s class show dev ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} diff --git a/tools/collector/scripts/etc.exclude b/tools/collector/scripts/etc.exclude index d3f3827d8..d870e9b22 100644 --- a/tools/collector/scripts/etc.exclude +++ b/tools/collector/scripts/etc.exclude @@ -35,6 +35,8 @@ /etc/bash_completion.d /etc/pm /etc/systemd/system/*.mount +/etc/systemd/system/*.socket +/etc/systemd/system/lvm2-lvmetad.service /etc/systemd/system/ctrl-alt-del.target /etc/ssl /etc/mtc/tmp diff --git a/tools/engtools/hostdata-collectors/patch-engtools.sh b/tools/engtools/hostdata-collectors/patch-engtools.sh index f64614279..dbb80682e 100755 --- a/tools/engtools/hostdata-collectors/patch-engtools.sh +++ b/tools/engtools/hostdata-collectors/patch-engtools.sh @@ -3,15 +3,15 @@ # http://twiki.wrs.com/PBUeng/Patching if [ -z $MY_WORKSPACE ] || [ -z $MY_REPO ]; then - echo "Some dev environment variables are not set." - echo "Refer to http://wiki.wrs.com/PBUeng/CentOSBuildProcess for instructions." - exit 1 + echo "Some dev environment variables are not set." + echo "Refer to http://wiki.wrs.com/PBUeng/CentOSBuildProcess for instructions." + exit 1 fi ENGTOOLS=$(ls ${MY_WORKSPACE}/std/rpmbuild/RPMS/engtools*noarch.rpm 2>/dev/null) if [ $? -ne 0 ]; then - echo "Engtools RPM has not been built. Please run \"build-pkgs engtools\" first." - exit 1 + echo "Engtools RPM has not been built. Please run \"build-pkgs engtools\" first." + exit 1 fi source ${MY_REPO}/stx/middleware/recipes-common/build-info/release-info.inc diff --git a/tools/engtools/hostdata-collectors/scripts/ceph.sh b/tools/engtools/hostdata-collectors/scripts/ceph.sh index 40014406f..61e5540d7 100644 --- a/tools/engtools/hostdata-collectors/scripts/ceph.sh +++ b/tools/engtools/hostdata-collectors/scripts/ceph.sh @@ -6,27 +6,26 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval OPT_USE_INTERVALS=1 # Print key ceph statistics -function print_ceph() -{ - print_separator - TOOL_HIRES_TIME +function print_ceph { + print_separator + TOOL_HIRES_TIME - cmd='ceph -s' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='ceph -s' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='ceph osd tree' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='ceph osd tree' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='ceph df detail' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='ceph df detail' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} } #------------------------------------------------------------------------------- @@ -47,10 +46,9 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do - print_ceph - sleep ${INTERVAL_SEC} +for ((rep=1; rep <= REPEATS ; rep++)); do + print_ceph + sleep ${INTERVAL_SEC} done print_ceph LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh b/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh index e1f16239d..5d5701b53 100644 --- a/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -ne 0 ]; then - ERRLOG "Require sudo/root access." - exit 1 + ERRLOG "Require sudo/root access." + exit 1 fi declare -a TOOLS @@ -32,25 +32,22 @@ LOG "Cleanup engtools:" # ( be sure not to clobber /etc/init.d/collect-engtools.sh ) LOG "kill processes brute force" pids=( $(pidof -x /usr/local/bin/collect-engtools.sh) ) -if [ ${#pids[@]} -ne 0 ] -then - LOG "killing: ${pids[@]}" - for pid in ${pids[@]} - do - LOG "kill: [ ${pid} ] " - pkill -KILL -P ${pid} - kill -9 ${pid} - done - pkill -KILL iostat - pkill -KILL top +if [ ${#pids[@]} -ne 0 ]; then + LOG "killing: ${pids[@]}" + for pid in ${pids[@]}; do + LOG "kill: [ ${pid} ] " + pkill -KILL -P ${pid} + kill -9 ${pid} + done + pkill -KILL iostat + pkill -KILL top else - LOG "no pids found" + LOG "no pids found" fi LOG "remove pidfiles" -for TOOL in "${TOOLS[@]}" -do - rm -f -v /var/run/${TOOL}.pid +for TOOL in "${TOOLS[@]}"; do + rm -f -v /var/run/${TOOL}.pid done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh b/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh index 7c1887a13..e59f382b4 100644 --- a/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh @@ -48,97 +48,92 @@ declare tgt_avail_bytes declare tgt_used_bytes # do_parallel_commands - launch parallel tools with separate output files -function do_parallel_commands() -{ - parallel_outfiles=() - for elem in "${tlist[@]}" - do - tool=""; period=""; repeat=""; interval="" - my_hash="elem[*]" - local ${!my_hash} - if [ ! -z "${name}" ]; then - fname="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${name}" - parallel_outfiles+=( $fname ) - LOG "collecting ${tool}, ${interval} second intervals, to: ${fname}" - if [ ! -z "${period}" ]; then - ${STDBUF} ${tool} -p ${period} -i ${interval} > ${fname} 2>/dev/null & - elif [ ! -z "${repeat}" ]; then - ${STDBUF} ${tool} --repeat=${repeat} --delay=${interval} > ${fname} 2>/dev/null & - fi - else - # run without file output (eg., ticker) - ${STDBUF} ${tool} -p ${period} -i ${interval} 2>/dev/null & - fi - done +function do_parallel_commands { + parallel_outfiles=() + for elem in "${tlist[@]}"; do + tool=""; period=""; repeat=""; interval="" + my_hash="elem[*]" + local ${!my_hash} + if [ ! -z "${name}" ]; then + fname="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${name}" + parallel_outfiles+=( $fname ) + LOG "collecting ${tool}, ${interval} second intervals, to: ${fname}" + if [ ! -z "${period}" ]; then + ${STDBUF} ${tool} -p ${period} -i ${interval} > ${fname} 2>/dev/null & + elif [ ! -z "${repeat}" ]; then + ${STDBUF} ${tool} --repeat=${repeat} --delay=${interval} > ${fname} 2>/dev/null & + fi + else + # run without file output (eg., ticker) + ${STDBUF} ${tool} -p ${period} -i ${interval} 2>/dev/null & + fi + done } # get_current_avail_usage() - get output destination file-system usage and # availability. # - updates: df_size_bytes, df_avail_bytes, du_used_bytes -function get_current_avail_usage() -{ - local -a df_arr_bytes=( $(df -P --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==2 {print $2, $4}') ) - df_size_bytes=${df_arr_bytes[0]} - df_avail_bytes=${df_arr_bytes[1]} - du_used_bytes=$(du --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==1 {print $1}') +function get_current_avail_usage { + local -a df_arr_bytes=( $(df -P --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==2 {print $2, $4}') ) + df_size_bytes=${df_arr_bytes[0]} + df_avail_bytes=${df_arr_bytes[1]} + du_used_bytes=$(du --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==1 {print $1}') } # purge_oldest_files() - remove oldest files based on file-system available space, # and maximum collection size -function purge_oldest_files() -{ +function purge_oldest_files { # get current file-system usage - get_current_avail_usage - msg=$(printf "avail %d MB, headroom %d MB; used %d MB, max %d MB" \ + get_current_avail_usage + msg=$(printf "avail %d MB, headroom %d MB; used %d MB, max %d MB" \ $[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] \ $[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024]) - LOG "usage: ${msg}" + LOG "usage: ${msg}" - if [[ $df_avail_bytes -lt $tgt_avail_bytes ]] || \ - [[ $du_used_bytes -gt $tgt_used_bytes ]]; then - # wait for compression to complete - wait + if [[ $df_avail_bytes -lt $tgt_avail_bytes ]] || \ + [[ $du_used_bytes -gt $tgt_used_bytes ]]; then + # wait for compression to complete + wait - get_current_avail_usage - if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then - msg=$(printf "purge: avail %d MB < target %d MB" \ - $[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] ) - LOG "purge: ${msg}" + get_current_avail_usage + if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then + msg=$(printf "purge: avail %d MB < target %d MB" \ + $[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] ) + LOG "purge: ${msg}" + fi + if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then + msg=$(printf "purge: used %d MB > target %d MB" \ + $[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024] ) + LOG "purge: ${msg}" + fi + else + return fi - if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then - msg=$(printf "purge: used %d MB > target %d MB" \ - $[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024] ) - LOG "purge: ${msg}" - fi - else - return - fi # remove files in oldest time sorted order until we meet usage targets, # incrementally updating usage as we remve files - for file in $( ls -rt ${TOOL_DEST_DIR}/${HOSTNAME}_* 2>/dev/null ) - do - if [[ $df_avail_bytes -ge $tgt_avail_bytes ]] && \ - [[ $du_used_bytes -le $tgt_used_bytes ]]; then - break - fi + for file in $( ls -rt ${TOOL_DEST_DIR}/${HOSTNAME}_* 2>/dev/null ); do + if [[ $df_avail_bytes -ge $tgt_avail_bytes ]] && \ + [[ $du_used_bytes -le $tgt_used_bytes ]]; then + break + fi - if [ ${OPT_DEBUG} -eq 1 ]; then - msg="purge: file=$file" - if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then - msg="${msg}, < AVAIL" - fi - if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then - msg="${msg}, > MAXUSAGE" - fi - LOG "${msg}" - fi + if [ ${OPT_DEBUG} -eq 1 ]; then + msg="purge: file=$file" + if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then + msg="${msg}, < AVAIL" + fi + if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then + msg="${msg}, > MAXUSAGE" + fi + LOG "${msg}" + fi - sz_bytes=$(stat --printf="%s" $file) - ((df_avail_bytes += sz_bytes)) - ((du_used_bytes -= sz_bytes)) - rm -fv ${file} - done + sz_bytes=$(stat --printf="%s" $file) + ((df_avail_bytes += sz_bytes)) + ((du_used_bytes -= sz_bytes)) + rm -fv ${file} + done } #------------------------------------------------------------------------------- @@ -160,23 +155,23 @@ CPULIST="" # Affine tools to NOVA pinned cores (i.e., non-cpu 0) # - remove interference with cpu 0 if [ "${AFFINE_PINNED}" -eq 1 ]; then - NOVA_CONF=/etc/nova/compute_extend.conf - if [ -f "${NOVA_CONF}" ]; then + NOVA_CONF=/etc/nova/compute_extend.conf + if [ -f "${NOVA_CONF}" ]; then source "${NOVA_CONF}" CPULIST=${compute_pinned_cpulist} - else + else CPULIST="" - fi + fi fi set_affinity ${CPULIST} # Define output directory if [[ "${HOSTNAME}" =~ "controller-" ]]; then - TOOL_DEST_DIR=/scratch/syseng_data/${HOSTNAME} + TOOL_DEST_DIR=/scratch/syseng_data/${HOSTNAME} elif [[ "${HOSTNAME}" =~ "compute-" ]]; then - TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} + TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} else - TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} + TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} fi mkdir -p ${TOOL_DEST_DIR} @@ -186,7 +181,7 @@ DAEMON_OUT="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${TOOLNAME}.log" # Redirect stdout and append to log if not connected to TTY if test ! -t 1 ; then - exec 1>> ${DAEMON_OUT} + exec 1>> ${DAEMON_OUT} fi # Get current availability and usage @@ -199,9 +194,9 @@ tgt_used_bytes=$[$PURGE_MAXUSAGE_MB*1024*1024] # Set granularity based on duration if [ $PERIOD_MIN -le 30 ]; then - GRAN_MIN=5 + GRAN_MIN=5 else - GRAN_MIN=60 + GRAN_MIN=60 fi # Adjust repeats and intervals based on GRAN_MIN granularity @@ -209,7 +204,7 @@ PERIOD_MIN=$[($PERIOD_MIN+(GRAN_MIN-1))/GRAN_MIN*GRAN_MIN] ((REPEATS = PERIOD_MIN/GRAN_MIN)) GRAN_MIN_IN_SEC=$[$GRAN_MIN*60] if [ ${INTERVAL_SEC} -gt ${GRAN_MIN_IN_SEC} ]; then - INTERVAL_SEC=${GRAN_MIN_IN_SEC} + INTERVAL_SEC=${GRAN_MIN_IN_SEC} fi # Define tools and options @@ -231,7 +226,7 @@ if [ ${OPT_SOAK} -eq 1 ]; then ceph_interval=${DUR_15MIN_IN_SEC} diskstats_interval=${DUR_15MIN_IN_SEC} memstats_interval=${DUR_15MIN_IN_SEC} - filestats_interval=${DUR_15MIN_IN_SEC} + filestats_interval=${DUR_15MIN_IN_SEC} elif [ ${OPT_SOAK} -eq 2 ]; then # Assume much shorter collection (eg, < hours) schedtop_interval=2 # i.e., 2 second interval @@ -243,7 +238,7 @@ elif [ ${OPT_SOAK} -eq 2 ]; then ceph_interval=${DUR_5MIN_IN_SEC} diskstats_interval=${DUR_5MIN_IN_SEC} memstats_interval=${DUR_5MIN_IN_SEC} - filestats_interval=${DUR_5MIN_IN_SEC} + filestats_interval=${DUR_5MIN_IN_SEC} else # Assume shorter collection (eg, < a few hours) schedtop_interval=5 # i.e., 5 second interval @@ -255,7 +250,7 @@ else ceph_interval=${DUR_5MIN_IN_SEC} diskstats_interval=${DUR_5MIN_IN_SEC} memstats_interval=${DUR_5MIN_IN_SEC} - filestats_interval=${DUR_5MIN_IN_SEC} + filestats_interval=${DUR_5MIN_IN_SEC} fi schedtop_repeat=$[ $PERIOD_MIN * 60 / $schedtop_interval ] occtop_repeat=$[ $PERIOD_MIN * 60 / $occtop_interval ] @@ -274,29 +269,29 @@ LBINDIR=/usr/local/bin declare -a tlist if [[ ${ENABLE_STATIC_COLLECTION} == "Y" ]] || [[ ${ENABLE_STATIC_COLLECTION} == "y" ]]; then - tlist+=( "tool=${LBINDIR}/top.sh name=top period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) - tlist+=( "tool=${LBINDIR}/iostat.sh name=iostat period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) - tlist+=( "tool=${LBINDIR}/netstats.sh name=netstats period=${PERIOD_MIN} interval=${netstats_interval}" ) - tlist+=( "tool=${BINDIR}/occtop name=occtop repeat=${occtop_repeat} interval=${occtop_interval}" ) - tlist+=( "tool=${BINDIR}/memtop name=memtop repeat=${memtop_repeat} interval=${memtop_interval}" ) - tlist+=( "tool=${BINDIR}/schedtop name=schedtop repeat=${schedtop_repeat} interval=${schedtop_interval}" ) - tlist+=( "tool=${LBINDIR}/diskstats.sh name=diskstats period=${PERIOD_MIN} interval=${diskstats_interval}" ) - tlist+=( "tool=${LBINDIR}/memstats.sh name=memstats period=${PERIOD_MIN} interval=${memstats_interval}" ) - tlist+=( "tool=${LBINDIR}/filestats.sh name=filestats period=${PERIOD_MIN} interval=${filestats_interval}" ) - if [[ "${HOSTNAME}" =~ "controller-" ]]; then - tlist+=( "tool=${LBINDIR}/ceph.sh name=ceph period=${PERIOD_MIN} interval=${ceph_interval}" ) - tlist+=( "tool=${LBINDIR}/postgres.sh name=postgres period=${PERIOD_MIN} interval=${postgres_interval}" ) - tlist+=( "tool=${LBINDIR}/rabbitmq.sh name=rabbitmq period=${PERIOD_MIN} interval=${rabbitmq_interval}" ) - elif [[ "${HOSTNAME}" =~ "compute-" ]]; then - tlist+=( "tool=${LBINDIR}/vswitch.sh name=vswitch period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) - fi + tlist+=( "tool=${LBINDIR}/top.sh name=top period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + tlist+=( "tool=${LBINDIR}/iostat.sh name=iostat period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + tlist+=( "tool=${LBINDIR}/netstats.sh name=netstats period=${PERIOD_MIN} interval=${netstats_interval}" ) + tlist+=( "tool=${BINDIR}/occtop name=occtop repeat=${occtop_repeat} interval=${occtop_interval}" ) + tlist+=( "tool=${BINDIR}/memtop name=memtop repeat=${memtop_repeat} interval=${memtop_interval}" ) + tlist+=( "tool=${BINDIR}/schedtop name=schedtop repeat=${schedtop_repeat} interval=${schedtop_interval}" ) + tlist+=( "tool=${LBINDIR}/diskstats.sh name=diskstats period=${PERIOD_MIN} interval=${diskstats_interval}" ) + tlist+=( "tool=${LBINDIR}/memstats.sh name=memstats period=${PERIOD_MIN} interval=${memstats_interval}" ) + tlist+=( "tool=${LBINDIR}/filestats.sh name=filestats period=${PERIOD_MIN} interval=${filestats_interval}" ) + if [[ "${HOSTNAME}" =~ "controller-" ]]; then + tlist+=( "tool=${LBINDIR}/ceph.sh name=ceph period=${PERIOD_MIN} interval=${ceph_interval}" ) + tlist+=( "tool=${LBINDIR}/postgres.sh name=postgres period=${PERIOD_MIN} interval=${postgres_interval}" ) + tlist+=( "tool=${LBINDIR}/rabbitmq.sh name=rabbitmq period=${PERIOD_MIN} interval=${rabbitmq_interval}" ) + elif [[ "${HOSTNAME}" =~ "compute-" ]]; then + tlist+=( "tool=${LBINDIR}/vswitch.sh name=vswitch period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + fi # ticker - shows progress on the screen - tlist+=( "tool=${LBINDIR}/ticker.sh name= period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + tlist+=( "tool=${LBINDIR}/ticker.sh name= period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) fi if [[ ${ENABLE_LIVE_STREAM} == "Y" ]] || [[ ${ENABLE_LIVE_STREAM} == "y" ]]; then - ${TOOLBIN}/live_stream.py & + ${TOOLBIN}/live_stream.py & fi #------------------------------------------------------------------------------- @@ -306,32 +301,30 @@ OPT_DEBUG=0 REP=0 if [ ${#tlist[@]} -ne 0 ]; then - # Static stats collection is turned on - while [[ ${TOOL_USR1_SIGNAL} -eq 0 ]] && - [[ ${OPT_FOREVER} -eq 1 || ${REP} -lt ${REPEATS} ]] - do - # increment loop counter - ((REP++)) + # Static stats collection is turned on + while [[ ${TOOL_USR1_SIGNAL} -eq 0 ]] && [[ ${OPT_FOREVER} -eq 1 || ${REP} -lt ${REPEATS} ]]; do + # increment loop counter + ((REP++)) - # purge oldest files - purge_oldest_files + # purge oldest files + purge_oldest_files - # define filename timestamp - timestamp=$( date +"%Y-%0m-%0e_%H%M" ) + # define filename timestamp + timestamp=$( date +"%Y-%0m-%0e_%H%M" ) - # collect tools in parallel to separate output files - LOG "collecting ${TOOLNAME} at ${timestamp} for ${PERIOD_MIN} mins, repeat=${REP}" - do_parallel_commands - wait + # collect tools in parallel to separate output files + LOG "collecting ${TOOLNAME} at ${timestamp} for ${PERIOD_MIN} mins, repeat=${REP}" + do_parallel_commands + wait + + # Compress latest increment + LOG "compressing: ${parallel_outfiles[@]}" + ${CMD_IDLE} bzip2 -q -f ${parallel_outfiles[@]} 2>/dev/null & + done - # Compress latest increment - LOG "compressing: ${parallel_outfiles[@]}" - ${CMD_IDLE} bzip2 -q -f ${parallel_outfiles[@]} 2>/dev/null & - done - # Wait for the compression to complete - wait - tools_cleanup 0 + wait + tools_cleanup 0 fi # Should wait here in case live stats streaming is turned on. diff --git a/tools/engtools/hostdata-collectors/scripts/diskstats.sh b/tools/engtools/hostdata-collectors/scripts/diskstats.sh index 376dbf185..c419a9a0e 100644 --- a/tools/engtools/hostdata-collectors/scripts/diskstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/diskstats.sh @@ -6,86 +6,84 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval OPT_USE_INTERVALS=1 # Print disk summary -function print_disk() -{ - print_separator - TOOL_HIRES_TIME +function print_disk { + print_separator + TOOL_HIRES_TIME # NOTES: # --total (grand-total) is a new option, but don't necessarily want to add tmpfs - # or dummy filesystems. + # or dummy filesystems. # - use -H to print in SI (eg, GB, vs GiB) # - can use -a to print all filesystems including dummy filesystems, but then # there can be double-counting: - print_separator - cmd='df -h -H -T --local -t ext2 -t ext3 -t ext4 -t xfs --total' - ${ECHO} "Disk space usage ext2,ext3,ext4,xfs,tmpfs (SI):" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='df -h -H -T --local -t ext2 -t ext3 -t ext4 -t xfs --total' + ${ECHO} "Disk space usage ext2,ext3,ext4,xfs,tmpfs (SI):" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='df -h -H -T --local -i -t ext2 -t ext3 -t ext4 -t xfs --total' - ${ECHO} "Disk inodes usage ext2,ext3,ext4,xfs,tmpfs (SI):" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='df -h -H -T --local -i -t ext2 -t ext3 -t ext4 -t xfs --total' + ${ECHO} "Disk inodes usage ext2,ext3,ext4,xfs,tmpfs (SI):" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='drbd-overview' - ${ECHO} "drbd disk usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='drbd-overview' + ${ECHO} "drbd disk usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='lvs' - ${ECHO} "logical volumes usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='lvs' + ${ECHO} "logical volumes usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='pvs' - ${ECHO} "physical volumes usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='pvs' + ${ECHO} "physical volumes usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='vgs' - ${ECHO} "volume groups usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='vgs' + ${ECHO} "volume groups usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} } # Print disk static summary -function print_disk_static() -{ - print_separator - cmd='cat /proc/scsi/scsi' - ${ECHO} "Attached devices: ${cmd}" - ${cmd} - ${ECHO} +function print_disk_static { + print_separator + cmd='cat /proc/scsi/scsi' + ${ECHO} "Attached devices: ${cmd}" + ${cmd} + ${ECHO} # fdisk - requires sudo/root - print_separator - cmd='fdisk -l' - if [ $UID -eq 0 ]; then - ${ECHO} "List disk devices: ${cmd}" - ${cmd} - else - WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" - fi - ${ECHO} + print_separator + cmd='fdisk -l' + if [ $UID -eq 0 ]; then + ${ECHO} "List disk devices: ${cmd}" + ${cmd} + else + WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" + fi + ${ECHO} # parted - requires sudo/root - print_separator - cmd='parted -l' - if [ $UID -eq 0 ]; then - ${ECHO} "List disk devices: ${cmd}" - ${cmd} - else - WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" - fi - ${ECHO} + print_separator + cmd='parted -l' + if [ $UID -eq 0 ]; then + ${ECHO} "List disk devices: ${cmd}" + ${cmd} + else + WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" + fi + ${ECHO} } #------------------------------------------------------------------------------- @@ -109,10 +107,9 @@ print_disk_static # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do - print_disk - sleep ${INTERVAL_SEC} +for ((rep=1; rep <= REPEATS ; rep++)); do + print_disk + sleep ${INTERVAL_SEC} done print_disk LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/engtools_util.sh b/tools/engtools/hostdata-collectors/scripts/engtools_util.sh index 311ccd2bf..af1aaa4e4 100644 --- a/tools/engtools/hostdata-collectors/scripts/engtools_util.sh +++ b/tools/engtools/hostdata-collectors/scripts/engtools_util.sh @@ -7,7 +7,7 @@ TOOL_USR1_SIGNAL=0 TOOL_USR2_SIGNAL=0 TOOL_TTY=0 if tty 1>/dev/null ; then - TOOL_TTY=1 + TOOL_TTY=1 fi # [ JGAULD : SHOULD RENAME TO TOOL_X ] @@ -19,145 +19,143 @@ CPULIST=0 # Include lsb functions if [ -d /lib/lsb ]; then -. /lib/lsb/init-functions + . /lib/lsb/init-functions else -. /etc/init.d/functions + . /etc/init.d/functions fi # Lightweight replacement for pidofproc -p -function check_pidfile () -{ - local pidfile pid +function check_pidfile { + local pidfile pid - OPTIND=1 - while getopts p: opt ; do + OPTIND=1 + while getopts p: opt ; do case "$opt" in - p) + p) pidfile="$OPTARG" ;; esac - done - shift $(($OPTIND - 1)) + done + shift $(($OPTIND - 1)) - read pid < "${pidfile}" - if [ -n "${pid:-}" ]; then - if $(kill -0 "${pid:-}" 2> /dev/null); then - echo "$pid" - return 0 - elif ps "${pid:-}" >/dev/null 2>&1; then - echo "$pid" - return 0 # program is running, but not owned by this user - else - return 1 # program is dead and /var/run pid file exists + read pid < "${pidfile}" + if [ -n "${pid:-}" ]; then + if $(kill -0 "${pid:-}" 2> /dev/null); then + echo "$pid" + return 0 + elif ps "${pid:-}" >/dev/null 2>&1; then + echo "$pid" + return 0 # program is running, but not owned by this user + else + return 1 # program is dead and /var/run pid file exists + fi fi - fi } # tools_init - initialize tool resources -function tools_init () -{ - local rc=0 - local error=0 - TOOLNAME=$(basename $0) +function tools_init { + local rc=0 + local error=0 + TOOLNAME=$(basename $0) # Check for sufficient priviledges - if [ $UID -ne 0 ]; then - ERRLOG "${NAME} requires sudo/root access." - return 1 - fi + if [ $UID -ne 0 ]; then + ERRLOG "${NAME} requires sudo/root access." + return 1 + fi # Check for essential binaries - ECHO=$(which echo 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ECHO=echo # use bash built-in echo - ${ECHO} "FATAL, 'echo' not found, rc=$rc"; - error=$rc - fi - DATE=$(which date 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ${ECHO} "FATAL, 'date' not found, rc=$rc"; - error=$rc - fi + ECHO=$(which echo 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ECHO=echo # use bash built-in echo + ${ECHO} "FATAL, 'echo' not found, rc=$rc"; + error=$rc + fi + DATE=$(which date 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ${ECHO} "FATAL, 'date' not found, rc=$rc"; + error=$rc + fi # Check for standard linux binaries, at least can use LOG functions now # - these are used in tools_header - CAT=$(which cat 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'cat' not found, rc=$rc"; - error=$rc - fi + CAT=$(which cat 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'cat' not found, rc=$rc"; + error=$rc + fi - ARCH=$(which arch 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'arch' not found, rc=$rc"; - error=$rc - fi + ARCH=$(which arch 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'arch' not found, rc=$rc"; + error=$rc + fi - SED=$(which sed 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'sed' not found, rc=$rc"; - error=$rc - fi + SED=$(which sed 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'sed' not found, rc=$rc"; + error=$rc + fi - GREP=$(which grep 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'grep' not found, rc=$rc"; - error=$rc - fi + GREP=$(which grep 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'grep' not found, rc=$rc"; + error=$rc + fi - WC=$(which wc 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'wc' not found, rc=$rc"; - error=$rc - fi + WC=$(which wc 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'wc' not found, rc=$rc"; + error=$rc + fi - UNAME=$(which uname 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'uname' not found, rc=$rc"; - error=$rc - fi + UNAME=$(which uname 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'uname' not found, rc=$rc"; + error=$rc + fi - SORT=$(which sort 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'sort' not found, rc=$rc"; - error=$rc - fi + SORT=$(which sort 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'sort' not found, rc=$rc"; + error=$rc + fi - TR=$(which tr 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'tr' not found, rc=$rc"; - error=$rc - fi + TR=$(which tr 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'tr' not found, rc=$rc"; + error=$rc + fi - AWK=$(which awk 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'awk' not found, rc=$rc"; - error=$rc - fi + AWK=$(which awk 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'awk' not found, rc=$rc"; + error=$rc + fi - PKILL=$(which pkill 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'pkill' not found, rc=$rc"; - error=$rc - fi + PKILL=$(which pkill 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'pkill' not found, rc=$rc"; + error=$rc + fi - LS=$(which ls 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'ls' not found, rc=$rc"; - error=$rc - fi + LS=$(which ls 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'ls' not found, rc=$rc"; + error=$rc + fi # The following block is needed for LSB systems such as Windriver Linux. # The utility is not available on CentOS so comment it out. @@ -170,168 +168,172 @@ function tools_init () # Let parent program decide what to do with the errors, # give ominous warning - if [ $error -eq 1 ]; then - WARNLOG "possibly cannot continue, missing linux binaries" - fi + if [ $error -eq 1 ]; then + WARNLOG "possibly cannot continue, missing linux binaries" + fi # Check if tool was previously running - if [ -e ${PIDFILE} ]; then + if [ -e ${PIDFILE} ]; then # [ JGAULD - remove pidofproc() / LSB compatibility issue ] if check_pidfile -p "${PIDFILE}" >/dev/null; then - ERRLOG "${PIDFILE} exists and ${TOOLNAME} is running" - return 1 + ERRLOG "${PIDFILE} exists and ${TOOLNAME} is running" + return 1 else # remove pid file - WARNLOG "${PIDFILE} exists but ${TOOLNAME} is not running; cleaning up" - rm -f ${PIDFILE} + WARNLOG "${PIDFILE} exists but ${TOOLNAME} is not running; cleaning up" + rm -f ${PIDFILE} + fi fi - fi # Create pid file - echo $$ > ${PIDFILE} + echo $$ > ${PIDFILE} # Setup trap handler - these signals trigger child shutdown and cleanup - trap tools_exit_handler INT HUP TERM EXIT - trap tools_usr1_handler USR1 - trap tools_usr2_handler USR2 + trap tools_exit_handler INT HUP TERM EXIT + trap tools_usr1_handler USR1 + trap tools_usr2_handler USR2 - return ${rc} + return ${rc} } # tools_cleanup() - terminate child processes -function tools_cleanup() { +function tools_cleanup { # restore signal handling to default behaviour - trap - INT HUP TERM EXIT - trap - USR1 USR2 + trap - INT HUP TERM EXIT + trap - USR1 USR2 - local VERBOSE_OPT='' - if [ "$1" -ne "0" ]; then - LOG "cleanup invoked with code: $1" - if [ ${TOOL_DEBUG} -ne 0 ]; then - VERBOSE_OPT='-v' + local VERBOSE_OPT='' + if [ "$1" -ne "0" ]; then + LOG "cleanup invoked with code: $1" + if [ ${TOOL_DEBUG} -ne 0 ]; then + VERBOSE_OPT='-v' + fi fi - fi # stop all processes launched from this process - pkill -TERM -P $$ - if [ "$1" -ne "0" ]; then - sleep 1 - fi + pkill -TERM -P $$ + if [ "$1" -ne "0" ]; then + sleep 1 + fi # OK, if the above didn't work, use force - pkill -KILL -P $$ + pkill -KILL -P $$ # remove pid file - if [ -e ${PIDFILE} ]; then - rm -f ${VERBOSE_OPT} ${PIDFILE} - fi - exit $1 + if [ -e ${PIDFILE} ]; then + rm -f ${VERBOSE_OPT} ${PIDFILE} + fi + exit $1 } # tools_exit_handler() - exit handler routine -function tools_exit_handler() { - TOOL_EXIT_SIGNAL=1 - tools_cleanup 128 +function tools_exit_handler { + TOOL_EXIT_SIGNAL=1 + tools_cleanup 128 } # tools_usr1_handler() - USR1 handler routine -function tools_usr1_handler() { - TOOL_USR1_SIGNAL=1 - LOG "caught USR1" +function tools_usr1_handler { + TOOL_USR1_SIGNAL=1 + LOG "caught USR1" } # tools_usr2_handler() - USR2 handler routine -function tools_usr2_handler() { - TOOL_USR2_SIGNAL=1 - LOG "caught USR1" +function tools_usr2_handler { + TOOL_USR2_SIGNAL=1 + LOG "caught USR1" } # LOG(), WARNLOG(), ERRLOG() - simple print log functions (not logger) -function LOG () -{ - local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) - echo "${tstamp_H} ${HOSTNAME} $0($$): $@"; +function LOG { + local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) + echo "${tstamp_H} ${HOSTNAME} $0($$): $@"; } -function LOG_NOCR () -{ - local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) - echo -n "${tstamp_H} ${HOSTNAME} $0($$): $@"; + +function LOG_NOCR { + local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) + echo -n "${tstamp_H} ${HOSTNAME} $0($$): $@"; +} + +function WARNLOG { + LOG "WARN $@"; +} + +function ERRLOG { + LOG "ERROR $@"; } -function WARNLOG () { LOG "WARN $@"; } -function ERRLOG () { LOG "ERROR $@"; } # TOOL_HIRES_TIME() - easily parsed date/timestamp and hi-resolution uptime -function TOOL_HIRES_TIME() -{ - echo "time: " $( ${DATE} +"%a %F %H:%M:%S.%N %Z %z" ) "uptime: " $( cat /proc/uptime ) +function TOOL_HIRES_TIME { + echo "time: " $( ${DATE} +"%a %F %H:%M:%S.%N %Z %z" ) "uptime: " $( cat /proc/uptime ) } # set_affinity() - set affinity for current script if a a CPULIST is defined -function set_affinity() { - local CPULIST=$1 - if [ -z "${CPULIST}" ]; then - return - fi +function set_affinity { + local CPULIST=$1 + if [ -z "${CPULIST}" ]; then + return + fi # Set cpu affinity for current program - local TASKSET=$(which taskset 2>/dev/null) - if [ -x "${TASKSET}" ]; then - ${TASKSET} -pc ${CPULIST} $$ 2>/dev/null - fi + local TASKSET=$(which taskset 2>/dev/null) + if [ -x "${TASKSET}" ]; then + ${TASKSET} -pc ${CPULIST} $$ 2>/dev/null + fi } # cmd_idle_priority() - command to set nice + ionice -function cmd_idle_priority() { - local NICE="" - local IONICE="" +function cmd_idle_priority { + local NICE="" + local IONICE="" - NICE=$( which nice 2>/dev/null ) - if [ $? -eq 0 ]; then - NICE="${NICE} -n 19" - else - NICE="" - fi - IONICE=$( which ionice 2>/dev/null ) - if [ $? -eq 0 ]; then - IONICE="${IONICE} -c 3" - else - IONICE="" - fi - echo "${NICE} ${IONICE}" + NICE=$( which nice 2>/dev/null ) + if [ $? -eq 0 ]; then + NICE="${NICE} -n 19" + else + NICE="" + fi + IONICE=$( which ionice 2>/dev/null ) + if [ $? -eq 0 ]; then + IONICE="${IONICE} -c 3" + else + IONICE="" + fi + echo "${NICE} ${IONICE}" } # print_separator() - print a horizontal separation line '\u002d' is '-' -function print_separator () { - printf '\u002d%.s' {1..80} - printf '\n' +function print_separator { + printf '\u002d%.s' {1..80} + printf '\n' } # tools_header() - print out common GenWare tools header -function tools_header() { - local TOOLNAME=$(basename $0) +function tools_header { + local TOOLNAME=$(basename $0) # Get timestamp #local tstamp=$( date +"%Y-%0m-%0e %H:%M:%S" 2>/dev/null ) - local tstamp=$( date --rfc-3339=ns | cut -c1-23 2>/dev/null ) + local tstamp=$( date --rfc-3339=ns | cut -c1-23 2>/dev/null ) # Linux Generic - local UPTIME=/proc/uptime + local UPTIME=/proc/uptime # Get number of online cpus - local CPUINFO=/proc/cpuinfo - local online_cpus=$( cat ${CPUINFO} | grep -i ^processor | wc -l 2>/dev/null ) + local CPUINFO=/proc/cpuinfo + local online_cpus=$( cat ${CPUINFO} | grep -i ^processor | wc -l 2>/dev/null ) # Get load average, run-queue size, and number of threads - local LOADAVG=/proc/loadavg - local LDAVG=( `cat ${LOADAVG} | sed -e 's#[/]# #g' 2>/dev/null` ) + local LOADAVG=/proc/loadavg + local LDAVG=( `cat ${LOADAVG} | sed -e 's#[/]# #g' 2>/dev/null` ) # Get current architecture - local arch=$( uname -m ) + local arch=$( uname -m ) # Determine processor name (there are many different formats... *sigh* ) # - build up info from multiple lines - local processor='unk' - local NAME=$( cat ${CPUINFO} | grep \ + local processor='unk' + local NAME=$( cat ${CPUINFO} | grep \ -e '^cpu\W\W:' \ -e ^'cpu model' \ -e ^'model name' \ @@ -340,139 +342,138 @@ function tools_header() { -e ^[Mm]achine | \ sort -u | awk 'BEGIN{FS=":";} {print $2;}' | \ tr '\n' ' ' | tr -s [:blank:] 2>/dev/null ) - if [ ! -z "${NAME}" ]; then - processor=${NAME} - fi + if [ ! -z "${NAME}" ]; then + processor=${NAME} + fi # Determine processor speed (abort grep after first match) - local speed='unk' - local BOGO=$( cat ${CPUINFO} | grep -m1 -e ^BogoMIPS -e ^bogomips | \ + local speed='unk' + local BOGO=$( cat ${CPUINFO} | grep -m1 -e ^BogoMIPS -e ^bogomips | \ awk 'BEGIN{FS=":";} {printf "%.1f", $2;}' 2>/dev/null ) - local MHZ=$( cat ${CPUINFO} | grep -m1 -e ^'cpu MHz' -e ^clock | \ + local MHZ=$( cat ${CPUINFO} | grep -m1 -e ^'cpu MHz' -e ^clock | \ awk 'BEGIN{FS=":";} {printf "%.1f", $2;}' 2>/dev/null ) - local MHZ2=$( cat ${CPUINFO} | grep -m1 -e ^Cpu0ClkTck -e ^'cycle frequency' | \ + local MHZ2=$( cat ${CPUINFO} | grep -m1 -e ^Cpu0ClkTck -e ^'cycle frequency' | \ awk 'BEGIN{FS=":";} {printf "%.1f", $2/1.0E6;}' 2>/dev/null ) - if [ ! -z "${MHZ}" ]; then - speed=${MHZ} - elif [ ! -z "${MHZ2}" ]; then - speed=${MHZ2} - elif [ ! -z ${BOGO} ]; then - speed=${BOGO} - fi + if [ ! -z "${MHZ}" ]; then + speed=${MHZ} + elif [ ! -z "${MHZ2}" ]; then + speed=${MHZ2} + elif [ ! -z ${BOGO} ]; then + speed=${BOGO} + fi # Determine OS and kernel version - local os_name=$( uname -s 2>/dev/null ) - local os_release=$( uname -r 2>/dev/null ) + local os_name=$( uname -s 2>/dev/null ) + local os_release=$( uname -r 2>/dev/null ) - declare -a arr + declare -a arr - local dist_id="" + local dist_id="" # Determine OS distribution ID - if [ lsb_pres == "yes" ]; then - arr=( $( lsb_release -i 2>/dev/null ) ) - dist_id=${arr[2]} - else - local dist_id=$(cat /etc/centos-release | awk '{print $1}' 2>/dev/null) - fi - - local dist_rel="" - if [ lsb_pres == "yes" ]; then + if [ lsb_pres == "yes" ]; then + arr=( $( lsb_release -i 2>/dev/null ) ) + dist_id=${arr[2]} + else + local dist_id=$(cat /etc/centos-release | awk '{print $1}' 2>/dev/null) + fi + + local dist_rel="" + if [ lsb_pres == "yes" ]; then # Determine OS distribution release - arr=( $( cat /proc/version | awk '{print $3}' 2>/dev/null ) ) - local dist_rel=${arr[1]} - else - local dist_rel=$(cat /etc/centos-release | awk '{print $4}' 2>/dev/null) - fi + arr=( $( cat /proc/version | awk '{print $3}' 2>/dev/null ) ) + local dist_rel=${arr[1]} + else + local dist_rel=$(cat /etc/centos-release | awk '{print $4}' 2>/dev/null) + fi # Print generic header - echo "${TOOLNAME} -- ${tstamp} load average:${LDAVG[0]}, ${LDAVG[1]}, ${LDAVG[2]} runq:${LDAVG[3]} nproc:${LDAVG[4]}" - echo " host:${HOSTNAME} Distribution:${dist_id} ${dist_rel} ${os_name} ${os_release}" - echo " arch:${arch} processor:${processor} speed:${speed} MHz CPUs:${online_cpus}" + echo "${TOOLNAME} -- ${tstamp} load average:${LDAVG[0]}, ${LDAVG[1]}, ${LDAVG[2]} runq:${LDAVG[3]} nproc:${LDAVG[4]}" + echo " host:${HOSTNAME} Distribution:${dist_id} ${dist_rel} ${os_name} ${os_release}" + echo " arch:${arch} processor:${processor} speed:${speed} MHz CPUs:${online_cpus}" } # tools_usage() - show generic tools tool usage -function tools_usage() { - if [ ${OPT_USE_INTERVALS} -eq 1 ]; then - echo "usage: ${TOOLNAME} [-f] [-p ] [-i ] [-c ] [-h]" - else - echo "Usage: ${TOOLNAME} [-f] [-p ] [-c ] [-h]" - fi +function tools_usage { + if [ ${OPT_USE_INTERVALS} -eq 1 ]; then + echo "usage: ${TOOLNAME} [-f] [-p ] [-i ] [-c ] [-h]" + else + echo "Usage: ${TOOLNAME} [-f] [-p ] [-c ] [-h]" + fi } # tools_print_help() - print generic tool help -function tools_print_help() { - tools_usage - echo - echo "Options:"; - echo " -f : collect forever : default: none" - echo " -p : overall collection period (minutes) : default: ${DEFAULT_PERIOD_MIN}" - if [ ${OPT_USE_INTERVALS} -eq 1 ]; then - echo " -i : sample interval (seconds) : default: ${DEFAULT_INTERVAL_SEC}" - fi - echo " -c : cpu list where tool runs (e.g., 0-1,8) : default: none" - echo - if [ ${OPT_USE_INTERVALS} -eq 1 ]; then - echo "Example: collect 5 minute period, sample every 30 seconds interval" - echo " ${TOOLNAME} -p 5 -i 30" - else - echo "Example: collect 5 minute period" - echo " ${TOOLNAME} -p 5" - fi +function tools_print_help { + tools_usage + echo + echo "Options:"; + echo " -f : collect forever : default: none" + echo " -p : overall collection period (minutes) : default: ${DEFAULT_PERIOD_MIN}" + if [ ${OPT_USE_INTERVALS} -eq 1 ]; then + echo " -i : sample interval (seconds) : default: ${DEFAULT_INTERVAL_SEC}" + fi + echo " -c : cpu list where tool runs (e.g., 0-1,8) : default: none" + echo + if [ ${OPT_USE_INTERVALS} -eq 1 ]; then + echo "Example: collect 5 minute period, sample every 30 seconds interval" + echo " ${TOOLNAME} -p 5 -i 30" + else + echo "Example: collect 5 minute period" + echo " ${TOOLNAME} -p 5" + fi } # tools_parse_options() -- parse common options for tools scripts -function tools_parse_options() { +function tools_parse_options { # check for no arguments, print usage - if [ $# -eq "0" ]; then - tools_usage - tools_cleanup 0 - exit 0 - fi + if [ $# -eq "0" ]; then + tools_usage + tools_cleanup 0 + exit 0 + fi # parse the input arguments - while getopts "fp:i:c:h" Option - do + while getopts "fp:i:c:h" Option; do case $Option in f) - OPT_FOREVER=1 - PERIOD_MIN=60 - ;; + OPT_FOREVER=1 + PERIOD_MIN=60 + ;; p) PERIOD_MIN=$OPTARG ;; i) - OPT_USE_INTERVALS=1 - INTERVAL_SEC=$OPTARG - ;; + OPT_USE_INTERVALS=1 + INTERVAL_SEC=$OPTARG + ;; c) CPULIST=$OPTARG ;; h) - tools_print_help - tools_cleanup 0 - exit 0 - ;; + tools_print_help + tools_cleanup 0 + exit 0 + ;; *) - tools_usage - tools_cleanup 0 - exit 0 - ;; + tools_usage + tools_cleanup 0 + exit 0 + ;; esac - done + done # validate input arguments - PERIOD_MAX=$[4*24*60] - INTERVAL_MAX=$[60*60] + PERIOD_MAX=$[4*24*60] + INTERVAL_MAX=$[60*60] - error=0 - if [[ ${PERIOD_MIN} -lt 1 || ${PERIOD_MIN} -gt ${PERIOD_MAX} ]]; then - echo "-p must be > 0 and <= ${PERIOD_MAX}." - error=1 - fi - if [[ ${INTERVAL_SEC} -lt 1 || ${INTERVAL_SEC} -gt ${INTERVAL_MAX} ]]; then - echo "-i must be > 0 and <= ${INTERVAL_MAX}." - error=1 - fi - if [ ${error} -eq 1 ]; then - tools_cleanup 0 - exit 1 - fi + error=0 + if [[ ${PERIOD_MIN} -lt 1 || ${PERIOD_MIN} -gt ${PERIOD_MAX} ]]; then + echo "-p must be > 0 and <= ${PERIOD_MAX}." + error=1 + fi + if [[ ${INTERVAL_SEC} -lt 1 || ${INTERVAL_SEC} -gt ${INTERVAL_MAX} ]]; then + echo "-i must be > 0 and <= ${INTERVAL_MAX}." + error=1 + fi + if [ ${error} -eq 1 ]; then + tools_cleanup 0 + exit 1 + fi } diff --git a/tools/engtools/hostdata-collectors/scripts/filestats.sh b/tools/engtools/hostdata-collectors/scripts/filestats.sh index 19d38a704..8b38e695b 100644 --- a/tools/engtools/hostdata-collectors/scripts/filestats.sh +++ b/tools/engtools/hostdata-collectors/scripts/filestats.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi PAGE_SIZE=$(getconf PAGE_SIZE) @@ -16,21 +16,20 @@ PAGE_SIZE=$(getconf PAGE_SIZE) OPT_USE_INTERVALS=1 -function print_files() -{ - print_separator +function print_files { + print_separator TOOL_HIRES_TIME - ${ECHO} "# ls -l /proc/*/fd" - sudo ls -l /proc/*/fd 2>/dev/null | awk \ - '$11 ~ /socket/ {a += 1} ; \ - $11 ~ /null/ {b += 1} ; \ - {c += 1} \ - END {\ - {printf "%-10s %-10s %-10s %-10s\n", "TOTAL", "FILES", "SOCKETS", "NULL PIPES"} \ - {printf "%-10s %-10s %-10s %-10s\n", c, c-(a+b) , a, b}}' + ${ECHO} "# ls -l /proc/*/fd" + sudo ls -l /proc/*/fd 2>/dev/null | awk \ + '$11 ~ /socket/ {a += 1} ; \ + $11 ~ /null/ {b += 1} ; \ + {c += 1} \ + END {\ + {printf "%-10s %-10s %-10s %-10s\n", "TOTAL", "FILES", "SOCKETS", "NULL PIPES"} \ + {printf "%-10s %-10s %-10s %-10s\n", c, c-(a+b) , a, b}}' - ${ECHO} + ${ECHO} ${ECHO} "# lsof" printf "%-7s %-7s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %s\n" "PID" "TOTAL" "FD" "U" "W" "R" "CWD" "RTD" "TXT" "MEM" "DEL" "TCP" "CMD" @@ -46,7 +45,7 @@ function print_files() {printf "%-7s %-7s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %s\n", \ pids[i]["PID"], \ pids[i]["TOTAL"],\ - ((pids[i]["u"]!="")? pids[i]["u"] : 0) + ((pids[i]["w"]!="")? pids[i]["w"] : 0 )+ ((pids[i]["r"]!="")? pids[i]["r"] : 0),\ + ((pids[i]["u"]!="")? pids[i]["u"] : 0) + ((pids[i]["w"]!="")? pids[i]["w"] : 0 )+ ((pids[i]["r"]!="")? pids[i]["r"] : 0),\ (pids[i]["u"]!="")? pids[i]["u"] : 0,\ (pids[i]["w"]!="")? pids[i]["w"] : 0,\ (pids[i]["r"]!="")? pids[i]["r"] : 0,\ @@ -56,13 +55,13 @@ function print_files() (pids[i]["mem"]!="")? pids[i]["mem"] : 0,\ (pids[i]["DEL"]!="")? pids[i]["DEL"] : 0,\ (pids[i]["TCP"]!="")? pids[i]["TCP"] : 0,\ - pids[i]["COMMAND"]} }}}' | sort -n -r -k3 + pids[i]["COMMAND"]} }}}' | sort -n -r -k3 - ${ECHO} + ${ECHO} - ${ECHO} "# lsof -nP +L1" - sudo lsof -nP +L1 - ${ECHO} + ${ECHO} "# lsof -nP +L1" + sudo lsof -nP +L1 + ${ECHO} } @@ -85,10 +84,9 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do - print_files - sleep ${INTERVAL_SEC} +for ((rep=1; rep <= REPEATS ; rep++)); do + print_files + sleep ${INTERVAL_SEC} done print_files LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh b/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh index 6712e7976..fa52a65a6 100644 --- a/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh @@ -41,76 +41,75 @@ fi # Check for sufficient priviledges # [ JGAULD : possibly provide user = 'operator' option instead... ] if [ $UID -ne 0 ]; then - log_daemon_msg "Starting ${NAME} requires sudo/root access." - exit 1 + log_daemon_msg "Starting ${NAME} requires sudo/root access." + exit 1 fi case $1 in - start) + start) if [ -e ${PIDFILE} ]; then - pid=$(pidof -x ${NAME}) - if test "${pid}" != "" - then - echo_success "${NAME} already running" + pid=$(pidof -x ${NAME}) + if test "${pid}" != "" + then + echo_success "${NAME} already running" exit - fi + fi fi log_daemon_msg "Starting ${NAME}" if start-stop-daemon --start --background --quiet --oknodo --pidfile ${PIDFILE} \ - --exec ${DAEMON} -- ${DAEMON_ARGS} ; then - log_end_msg 0 + --exec ${DAEMON} -- ${DAEMON_ARGS} ; then + log_end_msg 0 else - log_end_msg 1 + log_end_msg 1 fi ;; - stop) + stop) if [ -e ${PIDFILE} ]; then - pids=$(pidof -x ${NAME}) - if [[ ! -z "${pids}" ]] - then - echo_success "Stopping ${NAME} [$pid]" - start-stop-daemon --stop --quiet --oknodo --pidfile ${PIDFILE} --retry=TERM/3/KILL/5 - # [ JGAULD: none of the following should be necessary ] - /usr/local/bin/cleanup-engtools.sh - else - echo_failure "${NAME} is not running" - fi + pids=$(pidof -x ${NAME}) + if [[ ! -z "${pids}" ]]; then + echo_success "Stopping ${NAME} [$pid]" + start-stop-daemon --stop --quiet --oknodo --pidfile ${PIDFILE} --retry=TERM/3/KILL/5 + # [ JGAULD: none of the following should be necessary ] + /usr/local/bin/cleanup-engtools.sh + else + echo_failure "${NAME} is not running" + fi else - echo_failure "${PIDFILE} does not exist" + echo_failure "${PIDFILE} does not exist" fi ;; - restart) + restart) $0 stop && sleep 2 && $0 start ;; - status) + status) if [ -e ${PIDFILE} ]; then - pid=$(pidof -x ${NAME}) - if test "${pid}" != "" - then - echo_success "${NAME} is running" - else - echo_success "${NAME} is not running" - fi + pid=$(pidof -x ${NAME}) + if test "${pid}" != "" + then + echo_success "${NAME} is running" + else + echo_success "${NAME} is not running" + fi else - echo_success "${NAME} is not running" + echo_success "${NAME} is not running" fi ;; - reload) + reload) if [ -e ${PIDFILE} ]; then - start-stop-daemon --stop --signal USR1 --quiet --pidfile ${PIDFILE} --name ${NAME} - echo_success "${NAME} reloaded successfully" + start-stop-daemon --stop --signal USR1 --quiet --pidfile ${PIDFILE} --name ${NAME} + echo_success "${NAME} reloaded successfully" else - echo_success "${PIDFILE} does not exist" + echo_success "${PIDFILE} does not exist" fi ;; - *) + *) echo "Usage: $0 {start|stop|restart|reload|status}" exit 2 ;; diff --git a/tools/engtools/hostdata-collectors/scripts/iostat.sh b/tools/engtools/hostdata-collectors/scripts/iostat.sh index 04be90c35..7ac65aceb 100644 --- a/tools/engtools/hostdata-collectors/scripts/iostat.sh +++ b/tools/engtools/hostdata-collectors/scripts/iostat.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -15,9 +15,9 @@ OPT_USE_INTERVALS=1 IOSTAT=$( which iostat 2>/dev/null ) if [ $? -ne 0 ]; then - print_separator - WARNLOG "iostat not available" - tools_cleanup 0 + print_separator + WARNLOG "iostat not available" + tools_cleanup 0 fi # MAIN Program: diff --git a/tools/engtools/hostdata-collectors/scripts/linux_benchmark.sh b/tools/engtools/hostdata-collectors/scripts/linux_benchmark.sh index fb1d16d47..64fd5ce8b 100644 --- a/tools/engtools/hostdata-collectors/scripts/linux_benchmark.sh +++ b/tools/engtools/hostdata-collectors/scripts/linux_benchmark.sh @@ -29,8 +29,7 @@ compute_ports=(8000 8001 8002) traffic_types=(storage migration default drbd) flow_ids=(1:20 1:30 1:40 1:50) -function exec_cmd () -{ +function exec_cmd { node="$1" cmd="$2" @@ -41,8 +40,7 @@ function exec_cmd () fi } -function iperf3_server_start () -{ +function iperf3_server_start { local server="$1" local result="$2" local port="$3" @@ -55,8 +53,7 @@ function iperf3_server_start () $(exec_cmd "${server}" "${cmd}") } -function iperf3_client_tcp_start () -{ +function iperf3_client_tcp_start { local result="${result_dir}/throughput" local cmd="" local client="$1" @@ -76,8 +73,7 @@ function iperf3_client_tcp_start () $(exec_cmd "${client}" "${cmd} > ${result} 2>&1") } -function iperf3_client_udp_start () -{ +function iperf3_client_udp_start { local result="${result_dir}/throughput_udp" local cmd="" local client="$1" @@ -102,20 +98,17 @@ function iperf3_client_udp_start () $(exec_cmd "${client}" "${cmd} -b ${bw} >> ${result} 2>&1" ) } -function iperf3_stop () -{ +function iperf3_stop { local node="$1" local cmd="pkill iperf3" $(exec_cmd "${node}" "${cmd}") } -function get_ip_addr () -{ +function get_ip_addr { arp -a | grep -oP "(?<=$1 \()[^)]*" | head -n 1 } -function throughput_tcp_test() -{ +function throughput_tcp_test { for (( i = 0; i < ${#nodes[@]} ; i+=2 )); do for interface in "${interfaces[@]}"; do local interface_name="management" @@ -137,8 +130,7 @@ function throughput_tcp_test() done } -function throughput_udp_test () -{ +function throughput_udp_test { for (( i = 0; i < ${#nodes[@]} ; i+=2 )); do for interface in "${interfaces[@]}"; do local interface_name="management" @@ -229,8 +221,7 @@ function throughput_udp_test () done } -function throughput_parallel_test () -{ +function throughput_parallel_test { local dev="" local ip_addr="" local interface_name="" @@ -304,8 +295,7 @@ function throughput_parallel_test () done } -function latency_test () -{ +function latency_test { for (( i = 0; i < ${#nodes[@]} ; i+=2 )); do for interface in "${interfaces[@]}"; do local interface_name="management" @@ -328,16 +318,14 @@ function latency_test () done } -function setup () -{ +function setup { for node in ${nodes[@]}; do iperf3_stop "${node}" $(exec_cmd "${node}" "rm -rf ${result_dir}; mkdir -p ${result_dir}") done } -function get_remote_results () -{ +function get_remote_results { for node in ${nodes[@]}; do if [ "${node}" != "${host}" ]; then mkdir ${result_dir}/${node} @@ -346,8 +334,7 @@ function get_remote_results () done } -function get_interface_info () -{ +function get_interface_info { local dev="" local ip_addr="" printf "Network interfaces info\n" >> ${summary_file} @@ -365,8 +352,7 @@ function get_interface_info () done } -function generate_summary () -{ +function generate_summary { local header="" local result="" local result_file="" diff --git a/tools/engtools/hostdata-collectors/scripts/memstats.sh b/tools/engtools/hostdata-collectors/scripts/memstats.sh index 664b10b46..5ba340f0a 100644 --- a/tools/engtools/hostdata-collectors/scripts/memstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/memstats.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi PAGE_SIZE=$(getconf PAGE_SIZE) @@ -16,69 +16,68 @@ PAGE_SIZE=$(getconf PAGE_SIZE) OPT_USE_INTERVALS=1 # Print key networking device statistics -function print_memory() -{ +function print_memory { # Configuration for netcmds - MEMINFO=/proc/meminfo - NODEINFO=/sys/devices/system/node/node?/meminfo - BUDDYINFO=/proc/buddyinfo - SLABINFO=/proc/slabinfo + MEMINFO=/proc/meminfo + NODEINFO=/sys/devices/system/node/node?/meminfo + BUDDYINFO=/proc/buddyinfo + SLABINFO=/proc/slabinfo - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME - ${ECHO} "# ${MEMINFO}" - ${CAT} ${MEMINFO} - ${ECHO} + ${ECHO} "# ${MEMINFO}" + ${CAT} ${MEMINFO} + ${ECHO} - ${ECHO} "# ${NODEINFO}" - ${CAT} ${NODEINFO} - ${ECHO} + ${ECHO} "# ${NODEINFO}" + ${CAT} ${NODEINFO} + ${ECHO} - ${ECHO} "# ${BUDDYINFO}" - ${CAT} ${BUDDYINFO} - ${ECHO} + ${ECHO} "# ${BUDDYINFO}" + ${CAT} ${BUDDYINFO} + ${ECHO} - ${ECHO} "# PSS" - cat /proc/*/smaps 2>/dev/null | \ + ${ECHO} "# PSS" + cat /proc/*/smaps 2>/dev/null | \ awk '/^Pss:/ {a += $2;} END {printf "%d MiB\n", a/1024.0;}' - ${ECHO} + ${ECHO} # use old slabinfo format (i.e. slub not enabled in kernel) - ${ECHO} "# ${SLABINFO}" - ${CAT} ${SLABINFO} | \ + ${ECHO} "# ${SLABINFO}" + ${CAT} ${SLABINFO} | \ awk -v page_size_B=${PAGE_SIZE} ' BEGIN {page_KiB = page_size_B/1024; TOT_KiB = 0;} (NF == 17) { - gsub(/[<>]/, ""); - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", + gsub(/[<>]/, ""); + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", $2, $3, $4, $5, $6, $7, $8, $10, $11, $12, $13, $15, $16, $17, "KiB"); } (NF == 16) { - num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; - KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; - TOT_KiB += KiB; - printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", + num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; + KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; + TOT_KiB += KiB; + printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", $1, $2, $3, $4, $5, $6, $7, $9, $10, $11, $12, $14, $15, $16, KiB); } END { - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", "TOTAL", "-", "-", "-", "-", "-", ":", "-", "-", "-", ":", "-", "-", "-", TOT_KiB); } ' 2>/dev/null - ${ECHO} + ${ECHO} - ${ECHO} "# disk usage: rootfs, tmpfs" - cmd='df -h -H -T --local -t rootfs -t tmpfs' - ${ECHO} "Disk space usage rootfs,tmpfs (SI):" - ${ECHO} "${cmd}" - ${cmd} - ${ECHO} + ${ECHO} "# disk usage: rootfs, tmpfs" + cmd='df -h -H -T --local -t rootfs -t tmpfs' + ${ECHO} "Disk space usage rootfs,tmpfs (SI):" + ${ECHO} "${cmd}" + ${cmd} + ${ECHO} - CMD='ps -e -o ppid,pid,nlwp,rss:10,vsz:10,cmd --sort=-rss' - ${ECHO} "# ${CMD}" - ${CMD} - ${ECHO} + CMD='ps -e -o ppid,pid,nlwp,rss:10,vsz:10,cmd --sort=-rss' + ${ECHO} "# ${CMD}" + ${CMD} + ${ECHO} } #------------------------------------------------------------------------------- @@ -99,10 +98,9 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do - print_memory - sleep ${INTERVAL_SEC} +for ((rep=1; rep <= REPEATS ; rep++)); do + print_memory + sleep ${INTERVAL_SEC} done print_memory LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/netstats.sh b/tools/engtools/hostdata-collectors/scripts/netstats.sh index 4ed13b04e..c9be0a1e5 100644 --- a/tools/engtools/hostdata-collectors/scripts/netstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/netstats.sh @@ -6,33 +6,31 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval OPT_USE_INTERVALS=1 # Print key networking device statistics -function print_netcmds() -{ +function print_netcmds { # Configuration for netcmds - DEV=/proc/net/dev - NETSTAT=/proc/net/netstat + DEV=/proc/net/dev + NETSTAT=/proc/net/netstat - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME - for net in \ + for net in \ ${DEV} ${NETSTAT} - do - if [ -e "${net}" ] - then - ${ECHO} "# ${net}" - ${CAT} ${net} - ${ECHO} - fi - done + do + if [ -e "${net}" ]; then + ${ECHO} "# ${net}" + ${CAT} ${net} + ${ECHO} + fi + done } #------------------------------------------------------------------------------- @@ -53,10 +51,9 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do - print_netcmds - sleep ${INTERVAL_SEC} +for ((rep=1; rep <= REPEATS ; rep++)); do + print_netcmds + sleep ${INTERVAL_SEC} done print_netcmds LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/postgres.sh b/tools/engtools/hostdata-collectors/scripts/postgres.sh index 9bcf8d1f3..340d0e429 100644 --- a/tools/engtools/hostdata-collectors/scripts/postgres.sh +++ b/tools/engtools/hostdata-collectors/scripts/postgres.sh @@ -6,42 +6,40 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval OPT_USE_INTERVALS=1 # Print key networking device statistics -function print_postgres() -{ - print_separator - TOOL_HIRES_TIME - +function print_postgres { + print_separator + TOOL_HIRES_TIME + # postgressql command: set user, disable pagination, and be quiet - PSQL="sudo -u postgres psql --pset pager=off -q" + PSQL="sudo -u postgres psql --pset pager=off -q" # List postgres databases - db_list=( $(${PSQL} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) - ${ECHO} "# postgres databases" - echo "db_list = ${db_list[@]}" - ${ECHO} + db_list=( $(${PSQL} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) + ${ECHO} "# postgres databases" + echo "db_list = ${db_list[@]}" + ${ECHO} # List sizes of all postgres databases (similar to "\l+") - ${ECHO} "# postgres database sizes" - ${PSQL} -c " + ${ECHO} "# postgres database sizes" + ${PSQL} -c " SELECT - pg_database.datname, - pg_database_size(pg_database.datname), - pg_size_pretty(pg_database_size(pg_database.datname)) + pg_database.datname, + pg_database_size(pg_database.datname), + pg_size_pretty(pg_database_size(pg_database.datname)) FROM pg_database ORDER BY pg_database_size DESC; " # For each database, list tables and their sizes (similar to "\dt+") - for db in "${db_list[@]}" - do + for db in "${db_list[@]}"; do ${ECHO} "# postgres database: ${db}" ${PSQL} -d ${db} -c " SELECT @@ -85,29 +83,29 @@ SELECT last_autoanalyze FROM pg_stat_user_tables; " - done + done # Specific table counts (This is very SLOW, look at "live tuples" instead) # Number of keystone tokens #${ECHO} "# keystone token count" # Number of postgres connections - ${ECHO} "# postgres database connections" - CONN=$(ps -C postgres -o cmd= | wc -l) - CONN_T=$(ps -C postgres -o cmd= | awk '/postgres: / {print $3}' | awk '{for(i=1;i<=NF;i++) a[$i]++} END {for(k in a) print k, a[k]}' | sort -k 2 -nr ) - ${ECHO} "connections total = ${CONN}" - ${ECHO} - ${ECHO} "connections breakdown:" - ${ECHO} "${CONN_T}" - ${ECHO} + ${ECHO} "# postgres database connections" + CONN=$(ps -C postgres -o cmd= | wc -l) + CONN_T=$(ps -C postgres -o cmd= | awk '/postgres: / {print $3}' | awk '{for(i=1;i<=NF;i++) a[$i]++} END {for(k in a) print k, a[k]}' | sort -k 2 -nr ) + ${ECHO} "connections total = ${CONN}" + ${ECHO} + ${ECHO} "connections breakdown:" + ${ECHO} "${CONN_T}" + ${ECHO} - ${ECHO} "connections breakdown (query):" - ${PSQL} -c "SELECT datname,state,count(*) from pg_stat_activity group by datname,state;" - ${ECHO} + ${ECHO} "connections breakdown (query):" + ${PSQL} -c "SELECT datname,state,count(*) from pg_stat_activity group by datname,state;" + ${ECHO} - ${ECHO} "connections idle age:" - ${PSQL} -c "SELECT datname,age(now(),state_change) from pg_stat_activity where state='idle';" - ${ECHO} + ${ECHO} "connections idle age:" + ${PSQL} -c "SELECT datname,age(now(),state_change) from pg_stat_activity where state='idle';" + ${ECHO} } #------------------------------------------------------------------------------- @@ -128,10 +126,9 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do - print_postgres - sleep ${INTERVAL_SEC} +for ((rep=1; rep <= REPEATS ; rep++)); do + print_postgres + sleep ${INTERVAL_SEC} done print_postgres LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh b/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh index c588c16c5..2755024f6 100644 --- a/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh +++ b/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -15,10 +15,9 @@ OPT_USE_INTERVALS=1 #Need this workaround MQOPT="-n rabbit@localhost" # Print key networking device statistics -function print_rabbitmq() -{ - print_separator - TOOL_HIRES_TIME +function print_rabbitmq { + print_separator + TOOL_HIRES_TIME # IMPORTANT: # - Difficulty getting rabbitmqctl to work from init.d script; @@ -27,31 +26,31 @@ function print_rabbitmq() # - WORKAROUND: run command using 'sudo', even if you are 'root' # Dump various rabbitmq related stats - MQ_STATUS="rabbitmqctl ${MQOPT} status" - ${ECHO} "# ${MQ_STATUS}" - sudo ${MQ_STATUS} | grep -e '{memory' -A30 - ${ECHO} + MQ_STATUS="rabbitmqctl ${MQOPT} status" + ${ECHO} "# ${MQ_STATUS}" + sudo ${MQ_STATUS} | grep -e '{memory' -A30 + ${ECHO} # THe following is useful in diagnosing rabbit memory leaks # when end-users do not drain their queues (eg, due to RPC timeout issues, etc) - MQ_QUEUES="rabbitmqctl ${MQOPT} list_queues messages name pid messages_ready messages_unacknowledged memory consumers" - ${ECHO} "# ${MQ_QUEUES}" - sudo ${MQ_QUEUES} - ${ECHO} + MQ_QUEUES="rabbitmqctl ${MQOPT} list_queues messages name pid messages_ready messages_unacknowledged memory consumers" + ${ECHO} "# ${MQ_QUEUES}" + sudo ${MQ_QUEUES} + ${ECHO} - num_queues=$(sudo rabbitmqctl ${MQOPT} list_queues | wc -l); ((num_queues-=2)) - num_bindings=$(sudo rabbitmqctl ${MQOPT} list_bindings | wc -l); ((num_bindings-=2)) - num_exchanges=$(sudo rabbitmqctl ${MQOPT} list_exchanges | wc -l); ((num_exchanges-=2)) - num_connections=$(sudo rabbitmqctl ${MQOPT} list_connections | wc -l); ((num_connections-=2)) - num_channels=$(sudo rabbitmqctl ${MQOPT} list_channels | wc -l); ((num_channels-=2)) - arr=($(sudo rabbitmqctl ${MQOPT} list_queues messages consumers memory | \ - awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) - messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]} - printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" \ + num_queues=$(sudo rabbitmqctl ${MQOPT} list_queues | wc -l); ((num_queues-=2)) + num_bindings=$(sudo rabbitmqctl ${MQOPT} list_bindings | wc -l); ((num_bindings-=2)) + num_exchanges=$(sudo rabbitmqctl ${MQOPT} list_exchanges | wc -l); ((num_exchanges-=2)) + num_connections=$(sudo rabbitmqctl ${MQOPT} list_connections | wc -l); ((num_connections-=2)) + num_channels=$(sudo rabbitmqctl ${MQOPT} list_channels | wc -l); ((num_channels-=2)) + arr=($(sudo rabbitmqctl ${MQOPT} list_queues messages consumers memory | \ + awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) + messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]} + printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" \ "queues" "bindings" "exchanges" "connections" "channels" "messages" "consumers" "memory" - printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" \ + printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" \ $num_queues $num_bindings $num_exchanges $num_connections $num_channels $messages $consumers $memory - ${ECHO} + ${ECHO} } #------------------------------------------------------------------------------- @@ -72,10 +71,9 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do - print_rabbitmq - sleep ${INTERVAL_SEC} +for ((rep=1; rep <= REPEATS ; rep++)); do + print_rabbitmq + sleep ${INTERVAL_SEC} done print_rabbitmq LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh b/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh index 3d972c477..e5ccd1ab2 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -22,24 +22,24 @@ STORAGE=( $(system host-list | awk '(/storage/) {print $4;}') ) LOG "Remote bzip2 engtools data on all blades:" for blade in ${CONTROLLER[@]}; do - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "bzip2 on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo bzip2 /scratch/syseng_data/${blade}/* - else - WARNLOG "cannot ping: ${blade}" - fi + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "bzip2 on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo bzip2 /scratch/syseng_data/${blade}/* + else + WARNLOG "cannot ping: ${blade}" + fi done for blade in ${STORAGE[@]} ${COMPUTE[@]} ; do - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "bzip2 on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo bzip2 /tmp/syseng_data/${blade}/* - else - WARNLOG "cannot ping: ${blade}" - fi + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "bzip2 on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo bzip2 /tmp/syseng_data/${blade}/* + else + WARNLOG "cannot ping: ${blade}" + fi done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh b/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh index f3df76d55..21a47a4f3 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -18,19 +18,19 @@ BLADES=( $(system host-list | awk '(/compute|controller|storage/) {print $4;}') LOG "Remote start engtools on all blades:" for blade in ${BLADES[@]}; do - if [ "${blade}" == "${HOSTNAME}" ]; then - LOG "start on $blade:" - sudo service collect-engtools.sh start - else - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "start on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo service collect-engtools.sh start + if [ "${blade}" == "${HOSTNAME}" ]; then + LOG "start on $blade:" + sudo service collect-engtools.sh start else - WARNLOG "cannot ping: ${blade}" + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "start on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo service collect-engtools.sh start + else + WARNLOG "cannot ping: ${blade}" + fi fi - fi done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh b/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh index 1251ea827..5637bfd22 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -18,19 +18,19 @@ BLADES=( $(system host-list | awk '(/compute|controller|storage/) {print $4;}') LOG "Remote stop engtools on all blades:" for blade in ${BLADES[@]}; do - if [ "${blade}" == "${HOSTNAME}" ]; then - LOG "stop on $blade:" - sudo service collect-engtools.sh stop - else - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "stop on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo service collect-engtools.sh stop + if [ "${blade}" == "${HOSTNAME}" ]; then + LOG "stop on $blade:" + sudo service collect-engtools.sh stop else - WARNLOG "cannot ping: ${blade}" + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "stop on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo service collect-engtools.sh stop + else + WARNLOG "cannot ping: ${blade}" + fi fi - fi done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh b/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh index 6f82f47e7..095c5ddb7 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -24,10 +24,10 @@ STORAGE=( $(system host-list | awk '(/storage/) {print $4;}') ) DEST=/opt/backups/syseng_data/ if [[ "${HOSTNAME}" =~ "controller-" ]]; then - LOG "rsync DEST=${DEST}" + LOG "rsync DEST=${DEST}" else - LOG "*ERROR* only run this on controller" - exit 1 + LOG "*ERROR* only run this on controller" + exit 1 fi sudo mkdir -p ${DEST} @@ -41,29 +41,27 @@ LOG "rsync engtools data from all blades:" # controllers SRC=/scratch/syseng_data/ DEST=/opt/backups/syseng_data/ -for HOST in ${CONTROLLER[@]} -do - ping -c1 ${HOST} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" - sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} - else - WARNLOG "cannot ping: ${HOST}" - fi +for HOST in ${CONTROLLER[@]}; do + ping -c1 ${HOST} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" + sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} + else + WARNLOG "cannot ping: ${HOST}" + fi done # computes & storage SRC=/tmp/syseng_data/ DEST=/opt/backups/syseng_data/ -for HOST in ${STORAGE[@]} ${COMPUTE[@]} -do - ping -c1 ${HOST} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" - sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} - else - WARNLOG "cannot ping: ${HOST}" - fi +for HOST in ${STORAGE[@]} ${COMPUTE[@]}; do + ping -c1 ${HOST} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" + sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} + else + WARNLOG "cannot ping: ${HOST}" + fi done LOG 'done' diff --git a/tools/engtools/hostdata-collectors/scripts/slab.sh b/tools/engtools/hostdata-collectors/scripts/slab.sh index 70e9c0596..f60e0c095 100644 --- a/tools/engtools/hostdata-collectors/scripts/slab.sh +++ b/tools/engtools/hostdata-collectors/scripts/slab.sh @@ -3,19 +3,19 @@ PAGE_SIZE=$(getconf PAGE_SIZE) cat /proc/slabinfo | awk -v page_size_B=${PAGE_SIZE} ' BEGIN {page_KiB = page_size_B/1024; TOT_KiB = 0;} (NF == 17) { - gsub(/[<>]/, ""); - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", + gsub(/[<>]/, ""); + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", $2, $3, $4, $5, $6, $7, $8, $10, $11, $12, $13, $15, $16, $17, "KiB"); } (NF == 16) { - num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; - KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; - TOT_KiB += KiB; - printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", + num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; + KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; + TOT_KiB += KiB; + printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", $1, $2, $3, $4, $5, $6, $7, $9, $10, $11, $12, $14, $15, $16, KiB); } END { - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", "TOTAL", "-", "-", "-", "-", "-", ":", "-", "-", "-", ":", "-", "-", "-", TOT_KiB); } ' 2>/dev/null diff --git a/tools/engtools/hostdata-collectors/scripts/ticker.sh b/tools/engtools/hostdata-collectors/scripts/ticker.sh index 570cd526d..1edf00ed5 100644 --- a/tools/engtools/hostdata-collectors/scripts/ticker.sh +++ b/tools/engtools/hostdata-collectors/scripts/ticker.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -29,17 +29,16 @@ set_affinity ${CPULIST} LOG_NOCR "collecting " t=0 -for ((rep=1; rep <= REPEATS ; rep++)) -do - ((t++)) - sleep ${INTERVAL_SEC} - if [ ${t} -ge ${REP_LOG} ]; then - t=0 - echo "." - LOG_NOCR "collecting " - else - echo -n "." - fi +for ((rep=1; rep <= REPEATS ; rep++)); do + ((t++)) + sleep ${INTERVAL_SEC} + if [ ${t} -ge ${REP_LOG} ]; then + t=0 + echo "." + LOG_NOCR "collecting " + else + echo -n "." + fi done echo "." diff --git a/tools/engtools/hostdata-collectors/scripts/top.sh b/tools/engtools/hostdata-collectors/scripts/top.sh index 45dff33d7..9a5ebc0cd 100644 --- a/tools/engtools/hostdata-collectors/scripts/top.sh +++ b/tools/engtools/hostdata-collectors/scripts/top.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval diff --git a/tools/engtools/hostdata-collectors/scripts/vswitch.sh b/tools/engtools/hostdata-collectors/scripts/vswitch.sh index dae8bf76a..f0bad0eeb 100644 --- a/tools/engtools/hostdata-collectors/scripts/vswitch.sh +++ b/tools/engtools/hostdata-collectors/scripts/vswitch.sh @@ -6,35 +6,34 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval OPT_USE_INTERVALS=1 # Print key networking device statistics -function print_vswitch() -{ - print_separator - TOOL_HIRES_TIME +function print_vswitch { + print_separator + TOOL_HIRES_TIME - cmd='vshell engine-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell engine-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell port-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell port-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell network-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell network-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell interface-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell interface-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell engine-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell engine-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell port-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell port-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell network-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell network-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell interface-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell interface-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} } #------------------------------------------------------------------------------- @@ -55,10 +54,9 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do - print_vswitch - sleep ${INTERVAL_SEC} +for ((rep=1; rep <= REPEATS ; rep++)); do + print_vswitch + sleep ${INTERVAL_SEC} done print_vswitch LOG "done" diff --git a/tools/engtools/parsers/common/cleanup-uncompressed.sh b/tools/engtools/parsers/common/cleanup-uncompressed.sh index 3bd1fa21f..b810fe64f 100755 --- a/tools/engtools/parsers/common/cleanup-uncompressed.sh +++ b/tools/engtools/parsers/common/cleanup-uncompressed.sh @@ -5,15 +5,15 @@ #SPDX-License-Identifier: Apache-2.0 # # This script removes uncompressed file. It can save a huge amount of disk space -# on the analysis server. Run this script after the very last time the data is parsed -# and BEFORE running parse-daily.sh script. -# If it is run after each intermediary parse, the download-data.sh script will download the +# on the analysis server. Run this script after the very last time the data is parsed +# and BEFORE running parse-daily.sh script. +# If it is run after each intermediary parse, the download-data.sh script will download the # uncompressed files again. if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf @@ -22,8 +22,7 @@ YEAR=`date +'%Y'` files="${FILE_LIST// /, }" read -p "Are you sure you want to remove all uncompressed $files files? [Y/N]: " -n 1 -r echo -if [[ $REPLY =~ ^[Y]$ ]] -then +if [[ $REPLY =~ ^[Y]$ ]]; then for FILE in ${FILE_LIST}; do rm -v */*_${YEAR}-*${FILE} done diff --git a/tools/engtools/parsers/common/download-data.sh b/tools/engtools/parsers/common/download-data.sh index 0c5726dd1..7ac8c3d21 100755 --- a/tools/engtools/parsers/common/download-data.sh +++ b/tools/engtools/parsers/common/download-data.sh @@ -14,9 +14,9 @@ # This script is to be run after running download-computes.sh on one of the controllers. if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf @@ -38,6 +38,6 @@ for HOST in ${ALL_HOSTS}; do bzip2 ${HOST}* cd ${CURDIR} else - echo "${HOST} not found" + echo "${HOST} not found" fi done diff --git a/tools/engtools/parsers/common/parse-all.sh b/tools/engtools/parsers/common/parse-all.sh index 99d1e4eb5..754079693 100755 --- a/tools/engtools/parsers/common/parse-all.sh +++ b/tools/engtools/parsers/common/parse-all.sh @@ -8,7 +8,7 @@ # This script is used to parse all stats data. It is designed to be called by either # parse-controllers.sh or parse-computes.sh and not used as a standalone script. # If the input node is a controller, it will parse controller specific postgres & -# and rabbitmq stats first. If the input node is a compute, it will pars the compute +# and rabbitmq stats first. If the input node is a compute, it will pars the compute # specific vswitch stats first. # # The following parsing steps are common to all hosts and are executed in the specified order: @@ -22,12 +22,12 @@ # - Parse filestats (summary) # - Parse process level schedtop (optional step, configured in lab.conf) # - Generate tarball - + if [[ $# != 1 ]]; then - echo "ERROR: This script is meant to be called by either parse-controllers.sh or parse-computes.sh script." - echo "To run it separately, copy the script to the host directory that contains *.bz2 files." - echo "It takes a single argument - the name of the host directory (e.g. ./parse-all.sh controller-0)." - exit 1 + echo "ERROR: This script is meant to be called by either parse-controllers.sh or parse-computes.sh script." + echo "To run it separately, copy the script to the host directory that contains *.bz2 files." + echo "It takes a single argument - the name of the host directory (e.g. ./parse-all.sh controller-0)." + exit 1 fi source ../lab.conf @@ -41,17 +41,15 @@ NODE=$1 CURDATE=$(date) DATESTAMP=$(date +%b-%d) -function sedit() -{ +function sedit { local FILETOSED=$1 sed -i -e "s/ */ /g" ${FILETOSED} sed -i -e "s/ /,/g" ${FILETOSED} # Remove any trailing comma sed -i "s/,$//" ${FILETOSED} } - -function get_filename_from_mountname() -{ + +function get_filename_from_mountname { local name=$1 local fname if test "${name#*"scratch"}" != "${name}"; then @@ -82,8 +80,7 @@ function get_filename_from_mountname() echo $fname } -function parse_process_schedtop_data() -{ +function parse_process_schedtop_data { # Logic has been moved to a separate script so that parsing process level schedtop # can be run either as part of parse-all.sh script or independently. LOG "Process level schedtop parsing is turned on in lab.conf. Parsing schedtop detail..." @@ -92,8 +89,7 @@ function parse_process_schedtop_data() cd ${NODE} } -function parse_controller_specific() -{ +function parse_controller_specific { # Parsing Postgres data, removing data from previous run if there are any. Generate summary # data for each database and detail data for specified tables LOG "Parsing postgres data for ${NODE}" @@ -123,14 +119,12 @@ function parse_controller_specific() done } -function parse_compute_specific() -{ +function parse_compute_specific { LOG "Parsing vswitch data for ${NODE}" ../parse-vswitch.sh ${NODE} } -function parse_occtop_data() -{ +function parse_occtop_data { LOG "Parsing occtop data for ${NODE}" bzcat *occtop.bz2 >occtop-${NODE}-${DATESTAMP}.txt cp occtop-${NODE}-${DATESTAMP}.txt tmp.txt @@ -153,7 +147,7 @@ function parse_occtop_data() echo "${header}" > occtop-${NODE}-detailed.csv cat tmp2.txt >> occtop-${NODE}-detailed.csv - # Generate simple CSV file which is used to generate host CPU occupancy chart. Platform cores are + # Generate simple CSV file which is used to generate host CPU occupancy chart. Platform cores are # defined in the host.conf. The simple CSV contains only the Date/Time and Total platform CPU occupancy e.g. # Date/Time,Total # 2016-11-22 00:29:16.523,94.9 @@ -181,13 +175,12 @@ function parse_occtop_data() rm tmp.txt tmp2.txt tmpdate.txt tmpcore.txt } -function parse_memtop_data() -{ +function parse_memtop_data { LOG "Parsing memtop data for ${NODE}" bzcat *memtop.bz2 > memtop-${NODE}-${DATESTAMP}.txt cp memtop-${NODE}-${DATESTAMP}.txt tmp.txt sedit tmp.txt - + # After dumping all memtop bz2 output into one text file and in-place sed, grab only relevant data # for CSV output. Generate both detailed and simple CSV files. Simple output will be used to generate # chart. @@ -200,8 +193,7 @@ function parse_memtop_data() rm tmp.txt tmp2.txt } -function parse_netstats_data() -{ +function parse_netstats_data { LOG "Parsing netstats data for ${NODE}" # First generate the summary data then detail data for specified interfaces ../parse_netstats *netstats.bz2 > netstats-summary-${NODE}-${DATESTAMP}.txt @@ -222,11 +214,10 @@ function parse_netstats_data() done < tmp.txt done rm tmp.txt - fi + fi } -function parse_iostats_data() -{ +function parse_iostats_data { LOG "Parsing iostat data for ${NODE}" if [ -z "${IOSTATS_DEVICE_LIST}" ]; then ERRLOG "IOSTAT_DEVICE_LIST is not set in host.conf. Skipping iostats..." @@ -236,30 +227,29 @@ function parse_iostats_data() echo "Date/Time,${DEVICE},rqm/s,wrqm/s,r/s,w/s,rkB/s,wkB/s,avgrq-sz,avgqu-sz,await,r_await,w_await,svctm,%util" > iostat-${NODE}-${DEVICE}.csv # Dumping iostat content to tmp file bzcat *iostat.bz2 | grep -E "/2015|/2016|/2017|${DEVICE}" | awk '{print $1","$2","$3","$4","$5","$6","$7","$8","$9","$10","$11","$12","$13","$14}' > tmp.txt - while IFS= read -r current - do - if test "${current#*Linux}" != "$current" - then - # Skip the line that contains the word "Linux" - continue - else - if test "${current#*$DEVICE}" == "$current" - then - # It's a date entry, look ahead - read -r next - if test "${next#*$DEVICE}" != "${next}" - then - # This next line contains the device stats - # Combine date and time fields - current="${current//2016,/2016 }" - current="${current//2017,/2017 }" - # Combine time and AM/PM fields - current="${current//,AM/ AM}" - current="${current//,PM/ PM}" - # Write both lines to intermediate file - echo "${current}" >> tmp2.txt - echo "${next}" >> tmp2.txt - fi + while IFS= read -r current; do + if test "${current#*Linux}" != "$current" + then + # Skip the line that contains the word "Linux" + continue + else + if test "${current#*$DEVICE}" == "$current" + then + # It's a date entry, look ahead + read -r next + if test "${next#*$DEVICE}" != "${next}" + then + # This next line contains the device stats + # Combine date and time fields + current="${current//2016,/2016 }" + current="${current//2017,/2017 }" + # Combine time and AM/PM fields + current="${current//,AM/ AM}" + current="${current//,PM/ PM}" + # Write both lines to intermediate file + echo "${current}" >> tmp2.txt + echo "${next}" >> tmp2.txt + fi fi fi done < tmp.txt @@ -272,13 +262,12 @@ function parse_iostats_data() cut -d, -f2-11 --complement tmp2.txt > tmp.txt # Write final content to output csv cat tmp.txt >> iostat-${NODE}-${DEVICE}.csv - rm tmp.txt tmp2.txt + rm tmp.txt tmp2.txt done fi } -function parse_diskstats_data() -{ +function parse_diskstats_data { LOG "Parsing diskstats data for ${NODE}" if [ -z "${DISKSTATS_FILESYSTEM_LIST}" ]; then @@ -317,9 +306,9 @@ parse_occtop_data # Parsing memtop data parse_memtop_data -# Parsing memstats data to generate the high level report. The most important piece of info is the list of +# Parsing memstats data to generate the high level report. The most important piece of info is the list of # hi-runners at the end of the file. If there is a leak, run parse-daily.sh script to generate the time -# series data for the offending processes only. Use process name, not PID as most Titanium Cloud processes have +# series data for the offending processes only. Use process name, not PID as most Titanium Cloud processes have # workers. LOG "Parsing memstats summary for ${NODE}" ../parse_memstats --report *memstats.bz2 > memstats-summary-${NODE}-${DATESTAMP}.txt @@ -331,7 +320,7 @@ rm pid-*.csv parse_netstats_data # Parsing schedtop data to generate the high level report. Leave the process level schedtop parsing till -# the end as it is a long running task. +# the end as it is a long running task. LOG "Parsing schedtop summary for ${NODE}" FILES=$(ls *schedtop.bz2) ../parse_schedtop ${FILES} > schedtop-summary-${NODE}-${DATESTAMP}.txt @@ -342,17 +331,17 @@ parse_iostats_data # Parsing diskstats data parse_diskstats_data -# Parsing filestats data to generate the high level report. If there is a file descriptor leak, run parse-daily.sh -# script to generate the time series data for the offending processes only. Use process name, not PID as most +# Parsing filestats data to generate the high level report. If there is a file descriptor leak, run parse-daily.sh +# script to generate the time series data for the offending processes only. Use process name, not PID as most # Titanium Cloud processes have workers. LOG "Parsing filestats summary for ${NODE}" -../parse_filestats --all *filestats.bz2 > filestats-summary-${NODE}-${DATESTAMP}.txt +../parse_filestats --all *filestats.bz2 > filestats-summary-${NODE}-${DATESTAMP}.txt # Parsing process level schedtop data. This is a long running task. To skip this step or generate data for # only specific processes, update the lab.conf and host.conf files. [[ ${GENERATE_PROCESS_SCHEDTOP} == Y ]] && parse_process_schedtop_data || WARNLOG "Parsing process level schedtop is skipped." -# Done parsing for this host. If it's a controller host, check if the parsing of postgres connection stats which is run in +# Done parsing for this host. If it's a controller host, check if the parsing of postgres connection stats which is run in # parallel is done before creating a tar file. if test "${NODE#*"controller"}" != "${NODE}"; then # If postgres-conns.csv file has not been created which is highly unlikely, wait a couple of minutes @@ -362,8 +351,7 @@ if test "${NODE#*"controller"}" != "${NODE}"; then # is to use inotify which requires another inotify-tools package. oldsize=0 newsize=0 - while true - do + while true; do newsize=$(stat -c %s postgres-conns.csv) if [ "$oldsize" == "$newsize" ]; then break diff --git a/tools/engtools/parsers/common/parse-controllers.sh b/tools/engtools/parsers/common/parse-controllers.sh index 61a77c585..da52b07ea 100755 --- a/tools/engtools/parsers/common/parse-controllers.sh +++ b/tools/engtools/parsers/common/parse-controllers.sh @@ -14,9 +14,9 @@ PARSERDIR=$(dirname $0) . ${PARSERDIR}/parse-util.sh if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf @@ -36,12 +36,12 @@ for HOST in ${CONTROLLER_LIST}; do sleep 120 cd .. else - ERRLOG "${HOST} does not exist. Parsing skipped." + ERRLOG "${HOST} does not exist. Parsing skipped." fi done # Parsing postgres connection stats is a time consuming step, run it in parallel with parse-all -# script. +# script. for HOST in ${CONTROLLER_LIST}; do if [ -d ${HOST} ]; then LOG "Parsing postgres connection stats data for controller host ${HOST}" diff --git a/tools/engtools/parsers/common/parse-daily.sh b/tools/engtools/parsers/common/parse-daily.sh index 7298441c3..b8c70a5fe 100755 --- a/tools/engtools/parsers/common/parse-daily.sh +++ b/tools/engtools/parsers/common/parse-daily.sh @@ -19,16 +19,14 @@ # e.g. >./parse-daily.sh memstats sm-eru # >./parse-daily.sh controller-0 filestats postgress -function print_usage() -{ +function print_usage { echo "Usage: ./parse-daily.sh will parse daily data for all hosts." echo "Usage: ./parse-daily.sh will parse daily data for specified host." echo "Valid parsers for daily stats are: memstats & filestats." exit 1 } -function parse_daily_stats() -{ +function parse_daily_stats { local PARSER_NAME=$1 local PROCESS_NAME=$2 local TMPFILE="tmp.txt" @@ -102,8 +100,8 @@ elif [[ $# -eq 3 ]]; then cd $1 parse_daily_stats $2 $3 else - echo "ERROR: Specified host $1 does not exist." - exit 1 + echo "ERROR: Specified host $1 does not exist." + exit 1 fi else echo "Specified parser $2 is not a valid parser." diff --git a/tools/engtools/parsers/common/parse-postgres.sh b/tools/engtools/parsers/common/parse-postgres.sh index 2e15e12ca..982412336 100755 --- a/tools/engtools/parsers/common/parse-postgres.sh +++ b/tools/engtools/parsers/common/parse-postgres.sh @@ -25,8 +25,7 @@ else fi LOG "Parsing postgres files ${FILES}" -function generate_header() -{ +function generate_header { local header="Date/Time,Total" for DB in ${DATABASE_LIST}; do if [ ${DB} == "nova_api" ]; then @@ -59,18 +58,16 @@ function generate_header() echo $header } -function generate_grep_str() -{ +function generate_grep_str { local grepstr="time:" for DB in ${DATABASE_LIST}; do - grepstr="${grepstr}|${DB}" + grepstr="${grepstr}|${DB}" done grepstr="${grepstr}|breakdown|connections total|rows" echo $grepstr } -function init_variables() -{ +function init_variables { CONN_TOTAL="0" CONN_ACTIVE_TOTAL="0" CONN_IDLE_TOTAL="0" @@ -85,8 +82,7 @@ function init_variables() done } -function output_values() -{ +function output_values { local result="${DATEVAL} ${TIMEVAL},${CONN_TOTAL}" for DB in ${DATABASE_LIST}; do val=$(eval echo \${CONN_${DB^^}}) diff --git a/tools/engtools/parsers/common/parse-rabbitmq.sh b/tools/engtools/parsers/common/parse-rabbitmq.sh index a5f19473d..01df49a8e 100755 --- a/tools/engtools/parsers/common/parse-rabbitmq.sh +++ b/tools/engtools/parsers/common/parse-rabbitmq.sh @@ -8,7 +8,7 @@ OUTFILE=$1 FILES=$(ls *rabbitmq.bz2 | sort) -[ -e tmp.txt ] && rm tmp.txt +[ -e tmp.txt ] && rm tmp.txt echo "Time/Date,Memory Total,Connection Readers,Connection Writers,Connection Channels,Connection Other,Queue Procs,Queue Slave Procs,Plugins,Other Proc,Mnesia,Mgmt DB,Msg Index,Other ETS,Binary,Code,Atom,Other System,VM Memory High Watermark,VM Memory Limit,Disk Free Limit,Disk Free,Descriptor Limit,Descriptors Used,Sockets Limit,Sockets Used,Processes Limit,Processes Used,Run Queue,Uptime" >${OUTFILE} diff --git a/tools/engtools/parsers/common/parse-schedtop.sh b/tools/engtools/parsers/common/parse-schedtop.sh index 90cd88346..cd70586d4 100755 --- a/tools/engtools/parsers/common/parse-schedtop.sh +++ b/tools/engtools/parsers/common/parse-schedtop.sh @@ -17,16 +17,14 @@ PARSERDIR=$(dirname $0) . ${PARSERDIR}/parse-util.sh -function print_usage() -{ +function print_usage { echo "Usage: ./parse-schedtop.sh " echo " ./parse-schedtop.sh " echo "e.g. >./parse-schedtop.sh controller-0 nova-conductor" exit 1 } -function sedit() -{ +function sedit { local FILETOSED=$1 sed -i -e "s/ */ /g" ${FILETOSED} sed -i -e "s/ /,/2g" ${FILETOSED} @@ -34,8 +32,7 @@ function sedit() sed -i "s/,$//" ${FILETOSED} } -function parse_schedtop_data() -{ +function parse_schedtop_data { HOST=$1 SERVICE=$2 LOG "Parsing ${SERVICE} schedtop for host ${HOST}" @@ -48,7 +45,7 @@ function parse_schedtop_data() if [[ $# -eq 0 ]]; then # Parsing detail schedtop stats for all services configured in host.conf for all hosts would - # take a very long time and is often unnecessary. Until the performance issue with parse_schedtop + # take a very long time and is often unnecessary. Until the performance issue with parse_schedtop # is addressed, this is not supported. print_usage else diff --git a/tools/engtools/parsers/common/parse-util.sh b/tools/engtools/parsers/common/parse-util.sh index fa0c8f6d4..907da1ee7 100755 --- a/tools/engtools/parsers/common/parse-util.sh +++ b/tools/engtools/parsers/common/parse-util.sh @@ -8,19 +8,16 @@ #LOGFILE="${PARSERDIR}/parserlog.txt" LOGFILE="parserlog.txt" -function LOG () -{ - local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) - echo -e "${tstamp_H} $0($$): $@" >> ${LOGFILE} +function LOG { + local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) + echo -e "${tstamp_H} $0($$): $@" >> ${LOGFILE} } -function ERRLOG () -{ - LOG "ERROR: $@" +function ERRLOG { + LOG "ERROR: $@" } -function WARNLOG () -{ - LOG "WARN: $@" +function WARNLOG { + LOG "WARN: $@" } diff --git a/tools/engtools/parsers/large_office/download-computes.sh b/tools/engtools/parsers/large_office/download-computes.sh index 9e4806c01..fbceb79e3 100755 --- a/tools/engtools/parsers/large_office/download-computes.sh +++ b/tools/engtools/parsers/large_office/download-computes.sh @@ -6,14 +6,14 @@ # # Create /opt/backups/tmp/syseng-data directory on controller 0, change mode of this # directory to 777 and place this script and the lab.conf files there. It is recommended -# to set up password-less login from the controller to all storage and compute hosts +# to set up password-less login from the controller to all storage and compute hosts # before running the script. # if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf diff --git a/tools/engtools/parsers/large_office/parse-computes.sh b/tools/engtools/parsers/large_office/parse-computes.sh index 6addeaff4..60206c870 100755 --- a/tools/engtools/parsers/large_office/parse-computes.sh +++ b/tools/engtools/parsers/large_office/parse-computes.sh @@ -7,22 +7,22 @@ # This script is used to parse stats data for storage and compute hosts. It is not # relevant for CPE. For large office, it is called by parse-everything.sh script. -# File lab.conf must exist for the script to run. The STORAGE_LIST and COMPUTE_LIST -# config parameters can be set to suit the parsing needs. +# File lab.conf must exist for the script to run. The STORAGE_LIST and COMPUTE_LIST +# config parameters can be set to suit the parsing needs. PARSERDIR=$(dirname $0) . ${PARSERDIR}/parse-util.sh if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf if [ -z "${STORAGE_LIST}" ]; then - # This script could be invoked from another script or run separately so write to both + # This script could be invoked from another script or run separately so write to both # console and log file. echo "STORAGE_LIST is not set in lab.conf file. Skipping stats parsing for all storage." WARNLOG "STORAGE_LIST is not set in lab.conf file. Skipping stats parsing for all storage." @@ -35,7 +35,7 @@ else ../parse-all.sh ${HOST} > /dev/null 2>&1 & cd .. else - ERRLOG "${HOST} does not exist. Parsing skipped." + ERRLOG "${HOST} does not exist. Parsing skipped." fi done fi @@ -45,7 +45,7 @@ if [ -z "${COMPUTE_LIST}" ]; then WARNLOG "COMPUTE_LIST is not set in lab.conf file. Skipping stats parsing for all computes." exit 1 else - # If there is a large number of computes, they need to be parsed one batch at a time, + # If there is a large number of computes, they need to be parsed one batch at a time, # otherwise, the analysis server will be brought down to a crawl. Set the number of # computes to process in parallel as batches of 25 if it's not set in lab.conf BATCH_SIZE=${BATCH_SIZE:-25} diff --git a/tools/vm-topology/vm-topology/vm_topology/exec/vm_topology.py b/tools/vm-topology/vm-topology/vm_topology/exec/vm_topology.py index 3dbb32a7b..011900bce 100644 --- a/tools/vm-topology/vm-topology/vm_topology/exec/vm_topology.py +++ b/tools/vm-topology/vm-topology/vm_topology/exec/vm_topology.py @@ -444,7 +444,7 @@ def _mask_to_cpulist(mask=0): # Assume max number of cpus for now... max_cpus = 128 - for cpu in xrange(max_cpus): + for cpu in range(max_cpus): if ((1 << cpu) & mask): cpulist.append(cpu) return cpulist @@ -492,9 +492,9 @@ def range_to_list(csv_range=None): """ if not csv_range: return [] - xranges = [(lambda L: xrange(L[0], L[-1] + 1))(map(int, r.split('-'))) + ranges = [(lambda L: range(L[0], L[-1] + 1))(map(int, r.split('-'))) for r in csv_range.split(',')] - return [y for x in xranges for y in x] + return [y for x in ranges for y in x] class TimeoutError(Exception): @@ -613,7 +613,7 @@ def do_libvirt_domain_info((host)): cpulist_d = {} cpuset_total = 0 up_total = 0 - for vcpu in xrange(d_nrVirtCpu): + for vcpu in range(d_nrVirtCpu): cpuset_b = d_vcpus[1][vcpu] cpuset = 0 for cpu, up in enumerate(cpuset_b): diff --git a/tox.ini b/tox.ini index c2904c901..a1a548035 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,12 @@ deps = -r{toxinidir}/test-requirements.txt [testenv:linters] whitelist_externals = bash +#bashate ignore +#E006 Line too long +#E041 Arithmetic expansion using $[ is deprecated for $(( +#E042 local declaration hides errors +#E043 Arithmetic compound has inconsistent return semantics +#E044 Use [[ for non-POSIX comparisions commands = bash -c "find {toxinidir} \ -not \( -type d -name .?\* -prune \) \ @@ -21,10 +27,10 @@ commands = -not -name \*~ \ -not -name \*.md \ -name \*.sh \ - -print0 | xargs -0 bashate -v" + -print0 | xargs -0 bashate -v \ + -i E006,E041,E042,E043,E044 -e E*" bash -c "find {toxinidir} \ - \( -name middleware/io-monitor/recipes-common/io-monitor/io-monitor/io_monitor/test-tools/yaml/* -prune \) \ - -o \( -name .tox -prune \) \ + \( -path {toxinidir}/.tox \) -a -prune \ -o -type f -name '*.yaml' \ -print0 | xargs -0 yamllint" diff --git a/utilities/build-info/build-info-1.0/collect.sh b/utilities/build-info/build-info-1.0/collect.sh index 8d06f6a62..48abc23d0 100755 --- a/utilities/build-info/build-info-1.0/collect.sh +++ b/utilities/build-info/build-info-1.0/collect.sh @@ -24,7 +24,7 @@ if [ ! -e $jenkinsBuildFile ]; then fi if [ -e $releaseInfoFile ]; then - source $releaseInfoFile + source $releaseInfoFile fi if [ "${REPO}x" == "x" ]; then @@ -35,12 +35,12 @@ if [ -e $jenkinsBuildFile ]; then cp $jenkinsBuildFile $destFile source $jenkinsBuildFile else - # PLATFORM_RELEASE should be set in release-info.inc - if [ "x${PLATFORM_RELEASE}" == "x" ]; then - SW_VERSION="Unknown" - else - SW_VERSION="${PLATFORM_RELEASE}" - fi + # PLATFORM_RELEASE should be set in release-info.inc + if [ "x${PLATFORM_RELEASE}" == "x" ]; then + SW_VERSION="Unknown" + else + SW_VERSION="${PLATFORM_RELEASE}" + fi BUILD_TARGET="Unknown" BUILD_TYPE="Informal" @@ -89,8 +89,8 @@ fi echo "#ifndef _BUILD_INFO_H_" > $destH echo "#define _BUILD_INFO_H_" >> $destH echo "" >> $destH -echo "#define RELEASE_NAME \"$RELEASE_NAME\"" >> $destH -echo "#define SW_VERSION \"$SW_VERSION\"" >> $destH +echo "#define RELEASE_NAME \"$RELEASE_NAME\"" >> $destH +echo "#define SW_VERSION \"$SW_VERSION\"" >> $destH echo "" >> $destH echo "#define BUILD_TARGET \"$BUILD_TARGET\"" >> $destH echo "#define BUILD_TYPE \"$BUILD_TYPE\"" >> $destH diff --git a/utilities/platform-util/centos/build_srpm.data b/utilities/platform-util/centos/build_srpm.data index 8cae09b1e..c2ecd7719 100644 --- a/utilities/platform-util/centos/build_srpm.data +++ b/utilities/platform-util/centos/build_srpm.data @@ -1,4 +1,4 @@ SRC_DIR="platform-util" COPY_LIST_TO_TAR="scripts" -TIS_PATCH_VER=12 +TIS_PATCH_VER=13 diff --git a/utilities/platform-util/centos/platform-util.spec b/utilities/platform-util/centos/platform-util.spec index 12d9c510f..946b33da5 100644 --- a/utilities/platform-util/centos/platform-util.spec +++ b/utilities/platform-util/centos/platform-util.spec @@ -20,6 +20,12 @@ Summary: non controller platform utilities %description -n platform-util-noncontroller Platform utilities that don't get packaged on controller hosts +%package -n platform-util-controller +Summary: controller platform utilities + +%description -n platform-util-controller +Platform utilities that packaged on controllers or one node system + %define local_dir /usr/local %define local_bindir %{local_dir}/bin %define local_sbindir %{local_dir}/sbin @@ -52,6 +58,7 @@ install -m 700 -p -D %{_buildsubdir}/scripts/patch-restart-haproxy %{buildroot}% install -d %{buildroot}/etc/systemd/system install -m 644 -p -D %{_buildsubdir}/scripts/opt-platform.mount %{buildroot}/etc/systemd/system install -m 644 -p -D %{_buildsubdir}/scripts/opt-platform.service %{buildroot}/etc/systemd/system +install -m 644 -p -D %{_buildsubdir}/scripts/memcached.service %{buildroot}/etc/systemd/system # Mask the systemd ctrl-alt-delete.target, to disable reboot on ctrl-alt-del ln -sf /dev/null %{buildroot}/etc/systemd/system/ctrl-alt-del.target @@ -85,3 +92,7 @@ systemctl enable opt-platform.service # from parsing the fstab is not used by systemd. /etc/systemd/system/opt-platform.mount /etc/systemd/system/opt-platform.service + +%files -n platform-util-controller +%defattr(-,root,root,-) +/etc/systemd/system/memcached.service diff --git a/utilities/platform-util/scripts/cgcs_tc_setup.sh b/utilities/platform-util/scripts/cgcs_tc_setup.sh index 0630fac68..9dc3542af 100755 --- a/utilities/platform-util/scripts/cgcs_tc_setup.sh +++ b/utilities/platform-util/scripts/cgcs_tc_setup.sh @@ -22,17 +22,15 @@ fi # network link to autonegotiate link speed. Re-run the script in # the background so the parent can return right away and init can # continue. -if [ $# -eq 3 ] -then +if [ $# -eq 3 ]; then $0 $DEV $NETWORKTYPE $NETWORKSPEED dummy & disown exit 0 fi -function test_valid_speed -{ +function test_valid_speed { # After the link is enabled but before the autonegotiation is complete - # the link speed may be read as either -1 or as 4294967295 (which is + # the link speed may be read as either -1 or as 4294967295 (which is # uint(-1) in twos-complement) depending on the kernel. Neither one is valid. if (( $1 > 0 )) && (( $1 != 4294967295 )) then @@ -42,51 +40,43 @@ function test_valid_speed fi } -function log -{ +function log { # It seems that syslog isn't yet running, so append directly to the syslog file echo `date +%FT%T.%3N` `hostname` CGCS_TC_SETUP: $@ >> /var/log/platform.log } -function infra_exists -{ - if [ -z "$infrastructure_interface" ] - then +function infra_exists { + if [ -z "$infrastructure_interface" ]; then return 1 else return 0 fi } -function is_consolidated -{ +function is_consolidated { if ! infra_exists then - return 1 + return 1 else # determine whether the management interface is a parent of the # infrastructure interface based on name. # eg. this matches enp0s8 to enp0s8.10 but not enp0s88 - if [[ $infrastructure_interface =~ $management_interface[\.][0-9]+$ ]] - then - return 0 - fi - return 1 + if [[ $infrastructure_interface =~ $management_interface[\.][0-9]+$ ]]; then + return 0 + fi + return 1 fi } -function is_vlan -{ - if [ -f /proc/net/vlan/$DEV ] - then +function is_vlan { + if [ -f /proc/net/vlan/$DEV ]; then return 0 else return 1 fi } -function is_loopback -{ +function is_loopback { # (from include/uapi/linux/if.h) IFF_LOOPBACK=$((1<<3)) @@ -101,8 +91,7 @@ function is_loopback fi } -function setup_tc_port_filter -{ +function setup_tc_port_filter { local PORT=$1 local PORTMASK=$2 local FLOWID=$3 diff --git a/utilities/platform-util/scripts/memcached.service b/utilities/platform-util/scripts/memcached.service new file mode 100644 index 000000000..8dd1e891c --- /dev/null +++ b/utilities/platform-util/scripts/memcached.service @@ -0,0 +1,55 @@ +# +# This service file is a customized version in platform-util package from +# openstack/stx-integ project + +[Unit] +Description=memcached daemon +Before=httpd.service +After=network-online.target + +[Service] +EnvironmentFile=/etc/sysconfig/memcached +ExecStart=/usr/bin/memcached -p ${PORT} -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS + +# Set up a new file system namespace and mounts private /tmp and /var/tmp directories +# so this service cannot access the global directories and other processes cannot +# access this service's directories. +PrivateTmp=true + +# Mounts the /usr, /boot, and /etc directories read-only for processes invoked by this unit. +ProtectSystem=full + +# Ensures that the service process and all its children can never gain new privileges +NoNewPrivileges=true + +# Sets up a new /dev namespace for the executed processes and only adds API pseudo devices +# such as /dev/null, /dev/zero or /dev/random (as well as the pseudo TTY subsystem) to it, +# but no physical devices such as /dev/sda. +PrivateDevices=true + +# Required for dropping privileges and running as a different user +CapabilityBoundingSet=CAP_SETGID CAP_SETUID CAP_SYS_RESOURCE +LimitNOFILE=16384 + +# Attempts to create memory mappings that are writable and executable at the same time, +# or to change existing memory mappings to become executable are prohibited. +# XXX: this property is supported with systemd 231+ which is not yet on EL7 +# MemoryDenyWriteExecute=true + +# Restricts the set of socket address families accessible to the processes of this unit. +# Protects against vulnerabilities such as CVE-2016-8655 +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX + +# These service parameters are commented out since they are incompatible with +# Centos 7 and generate warning messages when included. +#ProtectKernelModules=true +#ProtectKernelTunables=true +#ProtectControlGroups=true +#RestrictRealtime=true +#RestrictNamespaces=true + +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/utilities/platform-util/scripts/remotelogging_tc_setup.sh b/utilities/platform-util/scripts/remotelogging_tc_setup.sh index 3f8f8e459..f70da1ec0 100755 --- a/utilities/platform-util/scripts/remotelogging_tc_setup.sh +++ b/utilities/platform-util/scripts/remotelogging_tc_setup.sh @@ -9,8 +9,7 @@ # $1 - listening port of remote log server PORT=$1 -function is_loopback -{ +function is_loopback { # (from include/uapi/linux/if.h) IFF_LOOPBACK=$((1<<3)) @@ -25,16 +24,14 @@ function is_loopback fi } -function log -{ +function log { # It seems that syslog isn't yet running, so append directly to the syslog file echo `date +%FT%T.%3N` `hostname` CGCS_TC_SETUP: $@ >> /var/log/platform.log } -function test_valid_speed -{ +function test_valid_speed { # After the link is enabled but before the autonegotiation is complete - # the link speed may be read as either -1 or as 4294967295 (which is + # the link speed may be read as either -1 or as 4294967295 (which is # uint(-1) in twos-complement) depending on the kernel. Neither one is valid. if (( $1 > 0 )) && (( $1 != 4294967295 )) then @@ -44,8 +41,7 @@ function test_valid_speed fi } -function get_dev_speed -{ +function get_dev_speed { # If the link doesn't come up we won't go enabled, so here we can # afford to wait forever for the link. while true diff --git a/virt/qemu/centos/files/build_configure.sh b/virt/qemu/centos/files/build_configure.sh index 31d78a19c..f9ba586f4 100755 --- a/virt/qemu/centos/files/build_configure.sh +++ b/virt/qemu/centos/files/build_configure.sh @@ -41,11 +41,11 @@ shift if [ "$have_rbd" == "enable" ]; then - rbd_driver=rbd, + rbd_driver=rbd, fi if [ "$have_gluster" == "enable" ]; then - gluster_driver=gluster, + gluster_driver=gluster, fi ./configure \