From 847cf1e92f9a966d6b03792fde7f3c3a5a40fce2 Mon Sep 17 00:00:00 2001 From: Sun Austin Date: Thu, 30 Aug 2018 14:50:33 +0800 Subject: [PATCH] Fix linters issue E002 and E003 and linters check (none-voting) E002 Tab indents E003 Indent not multiple of 4 Story: 2003366 Task: 24424 Change-Id: Ie063ccfe346489f08c4752adaa344f5919db0378 Signed-off-by: Sun Austin --- .zuul.yaml | 2 + base/haproxy/haproxy/haproxy.sh | 18 +- base/initscripts/centos/files/mountnfs.sh | 94 +-- .../centos/files/mysql-check-upgrade.sh | 6 +- .../centos/files/mysql-prepare-db-dir.sh | 4 +- .../centos/files/mysql-scripts-common.sh | 24 +- .../mariadb/centos/files/mysql-wait-ready.sh | 42 +- .../mariadb/centos/files/mysql-wait-stop.sh | 22 +- filesystem/nfscheck/files/nfscheck-init.sh | 4 +- .../lldpd/centos/files/i40e-lldp-configure.sh | 24 +- .../mlx4-config/files/mlx4-configure.sh | 12 +- .../mlx4-config/files/mlx4_core_config.sh | 2 +- tools/collector/scripts/collect_nfv_vim.sh | 2 +- tools/collector/scripts/collect_openstack.sh | 2 +- tools/collector/scripts/collect_psqldb.sh | 4 +- .../hostdata-collectors/patch-engtools.sh | 10 +- .../hostdata-collectors/scripts/ceph.sh | 24 +- .../scripts/cleanup-engtools.sh | 20 +- .../scripts/collect-engtools.sh | 234 ++++---- .../hostdata-collectors/scripts/diskstats.sh | 108 ++-- .../scripts/engtools_util.sh | 554 +++++++++--------- .../hostdata-collectors/scripts/filestats.sh | 40 +- .../scripts/init.d/collect-engtools.sh | 76 +-- .../hostdata-collectors/scripts/iostat.sh | 10 +- .../hostdata-collectors/scripts/memstats.sh | 84 +-- .../hostdata-collectors/scripts/netstats.sh | 34 +- .../hostdata-collectors/scripts/postgres.sh | 68 +-- .../hostdata-collectors/scripts/rabbitmq.sh | 50 +- .../scripts/remote/rbzip2-engtools.sh | 36 +- .../scripts/remote/rstart-engtools.sh | 26 +- .../scripts/remote/rstop-engtools.sh | 26 +- .../scripts/remote/rsync-engtools-data.sh | 38 +- .../hostdata-collectors/scripts/slab.sh | 14 +- .../hostdata-collectors/scripts/ticker.sh | 22 +- .../hostdata-collectors/scripts/top.sh | 4 +- .../hostdata-collectors/scripts/vswitch.sh | 44 +- .../parsers/common/cleanup-uncompressed.sh | 12 +- .../engtools/parsers/common/download-data.sh | 8 +- tools/engtools/parsers/common/parse-all.sh | 80 +-- .../parsers/common/parse-controllers.sh | 10 +- tools/engtools/parsers/common/parse-daily.sh | 4 +- .../engtools/parsers/common/parse-postgres.sh | 2 +- .../engtools/parsers/common/parse-rabbitmq.sh | 2 +- .../engtools/parsers/common/parse-schedtop.sh | 2 +- tools/engtools/parsers/common/parse-util.sh | 8 +- .../parsers/large_office/download-computes.sh | 8 +- .../parsers/large_office/parse-computes.sh | 16 +- .../build-info/build-info-1.0/collect.sh | 18 +- .../platform-util/scripts/cgcs_tc_setup.sh | 14 +- .../scripts/remotelogging_tc_setup.sh | 2 +- virt/qemu/centos/files/build_configure.sh | 4 +- 51 files changed, 988 insertions(+), 986 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 03a8bd5c8..549078afd 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -4,3 +4,5 @@ jobs: - openstack-tox-pep8: voting: false + - openstack-tox-linters: + voting: false diff --git a/base/haproxy/haproxy/haproxy.sh b/base/haproxy/haproxy/haproxy.sh index 560480e40..dc7e6ea92 100755 --- a/base/haproxy/haproxy/haproxy.sh +++ b/base/haproxy/haproxy/haproxy.sh @@ -7,7 +7,7 @@ # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: HA-Proxy TCP/HTTP reverse proxy -# Description: HA-Proxy is a TCP/HTTP reverse proxy +# Description: HA-Proxy is a TCP/HTTP reverse proxy ### END INIT INFO PATH=/sbin:/bin:/usr/sbin:/usr/bin @@ -22,12 +22,12 @@ RETVAL=0 # This is only needed till TPM In-Kernel # ResourceMgr comes in remove_TPM_transients () { - _HANDLES=`find $TPM_DATA_DIR -type f -name "hp*.bin" -printf "%f "` - for handle in $_HANDLES; do + _HANDLES=`find $TPM_DATA_DIR -type f -name "hp*.bin" -printf "%f "` + for handle in $_HANDLES; do handle_addr=`echo $handle | sed 's/hp\([0-9]*\)\.bin/\1/g'` tss2_flushcontext -ha $handle_addr &> /dev/null - done - rm -f $TPM_DATA_DIR/* + done + rm -f $TPM_DATA_DIR/* } start() { @@ -60,9 +60,9 @@ start() { stop() { if [ ! -e $PIDFILE ]; then return; fi - + echo -n "Stopping $DESC..." - + start-stop-daemon --stop --quiet --retry 3 --oknodo --pidfile $PIDFILE -x "$DAEMON" if [ -n "`pidof $DAEMON`" ] ; then pkill -KILL -f $DAEMON @@ -90,7 +90,7 @@ status() } check() { - /usr/sbin/$NAME -c -q -V -f /etc/$NAME/$NAME.cfg + /usr/sbin/$NAME -c -q -V -f /etc/$NAME/$NAME.cfg } # See how we were called. @@ -116,5 +116,5 @@ case "$1" in RETVAL=1 ;; esac - + exit $RETVAL diff --git a/base/initscripts/centos/files/mountnfs.sh b/base/initscripts/centos/files/mountnfs.sh index acc07797b..8d55c9175 100755 --- a/base/initscripts/centos/files/mountnfs.sh +++ b/base/initscripts/centos/files/mountnfs.sh @@ -15,14 +15,14 @@ if [ "$1" = "stop" ]; then fi # -# Run in a subshell because of I/O redirection. +# Run in a subshell because of I/O redirection. # test -f /etc/fstab && ( # -# Read through fstab line by line. If it is NFS, set the flag -# for mounting NFS filesystems. If any NFS partition is found and it -# not mounted with the nolock option, we start the rpcbind. +# Read through fstab line by line. If it is NFS, set the flag +# for mounting NFS filesystems. If any NFS partition is found and it +# not mounted with the nolock option, we start the rpcbind. # rpcbind=no mount_nfs=no @@ -31,41 +31,41 @@ mount_ncp=no mount_cifs=no while read device mountpt fstype options do - case "$device" in - ""|\#*) - continue - ;; - esac + case "$device" in + ""|\#*) + continue + ;; + esac - case "$options" in - *noauto*) - continue - ;; - esac + case "$options" in + *noauto*) + continue + ;; + esac - if test "$fstype" = nfs - then - mount_nfs=yes - case "$options" in - *nolock*) - ;; - *) - rpcbind=yes - ;; - esac - fi - if test "$fstype" = smbfs - then - mount_smb=yes - fi - if test "$fstype" = ncpfs - then - mount_ncp=yes - fi - if test "$fstype" = cifs - then - mount_cifs=yes - fi + if test "$fstype" = nfs + then + mount_nfs=yes + case "$options" in + *nolock*) + ;; + *) + rpcbind=yes + ;; + esac + fi + if test "$fstype" = smbfs + then + mount_smb=yes + fi + if test "$fstype" = ncpfs + then + mount_ncp=yes + fi + if test "$fstype" = cifs + then + mount_cifs=yes + fi done exec 0>&1 @@ -78,11 +78,11 @@ then then if test -x /usr/sbin/rpcbind then - service rpcbind status > /dev/null - if [ $? != 0 ]; then - echo -n "Starting rpcbind..." - start-stop-daemon --start --quiet --exec /usr/sbin/rpcbind - sleep 2 + service rpcbind status > /dev/null + if [ $? != 0 ]; then + echo -n "Starting rpcbind..." + start-stop-daemon --start --quiet --exec /usr/sbin/rpcbind + sleep 2 fi fi fi @@ -90,11 +90,11 @@ fi if test "$mount_nfs" = yes || test "$mount_smb" = yes || test "$mount_ncp" = yes || test "$mount_cifs" = yes then - echo "Mounting remote filesystems..." - test "$mount_nfs" = yes && mount -a -t nfs - test "$mount_smb" = yes && mount -a -t smbfs - test "$mount_ncp" = yes && mount -a -t ncpfs - test "$mount_cifs" = yes && mount -a -t cifs + echo "Mounting remote filesystems..." + test "$mount_nfs" = yes && mount -a -t nfs + test "$mount_smb" = yes && mount -a -t smbfs + test "$mount_ncp" = yes && mount -a -t ncpfs + test "$mount_cifs" = yes && mount -a -t cifs fi ) < /etc/fstab diff --git a/database/mariadb/centos/files/mysql-check-upgrade.sh b/database/mariadb/centos/files/mysql-check-upgrade.sh index 1bfd3bc6a..cd669bda8 100644 --- a/database/mariadb/centos/files/mysql-check-upgrade.sh +++ b/database/mariadb/centos/files/mysql-check-upgrade.sh @@ -27,9 +27,9 @@ if [ $version -ne $thisversion ] ; then cat <&2 The datadir located at $datadir needs to be upgraded using 'mysql_upgrade' tool. This can be done using the following steps: - 1. Back-up your data before with 'mysql_upgrade' - 2. Start the database daemon using 'service @DAEMON_NAME@ start' - 3. Run 'mysql_upgrade' with a database user that has sufficient privileges + 1. Back-up your data before with 'mysql_upgrade' + 2. Start the database daemon using 'service @DAEMON_NAME@ start' + 3. Run 'mysql_upgrade' with a database user that has sufficient privileges Read more about 'mysql_upgrade' usage at: https://mariadb.com/kb/en/mariadb/documentation/sql-commands/table-commands/mysql_upgrade/ diff --git a/database/mariadb/centos/files/mysql-prepare-db-dir.sh b/database/mariadb/centos/files/mysql-prepare-db-dir.sh index b47fa9b4a..96e899813 100644 --- a/database/mariadb/centos/files/mysql-prepare-db-dir.sh +++ b/database/mariadb/centos/files/mysql-prepare-db-dir.sh @@ -41,14 +41,14 @@ else fi myuser=`systemctl show -p User "${SERVICE_NAME}" | - sed 's/^User=//'` + sed 's/^User=//'` if [ x"$myuser" = x ] then myuser=mysql fi mygroup=`systemctl show -p Group "${SERVICE_NAME}" | - sed 's/^Group=//'` + sed 's/^Group=//'` if [ x"$mygroup" = x ] then mygroup=mysql diff --git a/database/mariadb/centos/files/mysql-scripts-common.sh b/database/mariadb/centos/files/mysql-scripts-common.sh index bf2888f2a..f77a8040b 100644 --- a/database/mariadb/centos/files/mysql-scripts-common.sh +++ b/database/mariadb/centos/files/mysql-scripts-common.sh @@ -11,18 +11,18 @@ export LC_ALL=C # We use my_print_defaults which prints all options from multiple files, # with the more specific ones later; hence take the last match. get_mysql_option(){ - if [ $# -ne 3 ] ; then - echo "get_mysql_option requires 3 arguments: section option default_value" - return - fi - sections="$1" - option_name="$2" - default_value="$3" - result=`@bindir@/my_print_defaults $sections | sed -n "s/^--${option_name}=//p" | tail -n 1` - if [ -z "$result" ]; then - # not found, use default - result="${default_value}" - fi + if [ $# -ne 3 ] ; then + echo "get_mysql_option requires 3 arguments: section option default_value" + return + fi + sections="$1" + option_name="$2" + default_value="$3" + result=`@bindir@/my_print_defaults $sections | sed -n "s/^--${option_name}=//p" | tail -n 1` + if [ -z "$result" ]; then + # not found, use default + result="${default_value}" + fi } # Defaults here had better match what mysqld_safe will default to diff --git a/database/mariadb/centos/files/mysql-wait-ready.sh b/database/mariadb/centos/files/mysql-wait-ready.sh index 2ed5fe1d3..6d95800aa 100644 --- a/database/mariadb/centos/files/mysql-wait-ready.sh +++ b/database/mariadb/centos/files/mysql-wait-ready.sh @@ -9,8 +9,8 @@ source "`dirname ${BASH_SOURCE[0]}`/mysql-scripts-common" # "After mysqld" won't be started until the database is really ready. if [ $# -ne 1 ] ; then - echo "You need to pass daemon pid as an argument for this script." - exit 20 + echo "You need to pass daemon pid as an argument for this script." + exit 20 fi # Service file passes us the daemon's PID (actually, mysqld_safe's PID) @@ -19,27 +19,27 @@ daemon_pid="$1" # Wait for the server to come up or for the mysqld process to disappear ret=0 while /bin/true; do - # Check process still exists - if ! [ -d "/proc/${daemon_pid}" ] ; then - ret=1 - break - fi - RESPONSE=`@bindir@/mysqladmin --no-defaults --socket="$socketfile" --user=UNKNOWN_MYSQL_USER ping 2>&1` - mret=$? - if [ $mret -eq 0 ] ; then - break - fi - # exit codes 1, 11 (EXIT_CANNOT_CONNECT_TO_SERVICE) are expected, - # anything else suggests a configuration error - if [ $mret -ne 1 -a $mret -ne 11 ]; then + # Check process still exists + if ! [ -d "/proc/${daemon_pid}" ] ; then + ret=1 + break + fi + RESPONSE=`@bindir@/mysqladmin --no-defaults --socket="$socketfile" --user=UNKNOWN_MYSQL_USER ping 2>&1` + mret=$? + if [ $mret -eq 0 ] ; then + break + fi + # exit codes 1, 11 (EXIT_CANNOT_CONNECT_TO_SERVICE) are expected, + # anything else suggests a configuration error + if [ $mret -ne 1 -a $mret -ne 11 ]; then echo "Cannot check for @NICE_PROJECT_NAME@ Daemon startup because of mysqladmin failure." >&2 - ret=$mret - break - fi - # "Access denied" also means the server is alive - echo "$RESPONSE" | grep -q "Access denied for user" && break + ret=$mret + break + fi + # "Access denied" also means the server is alive + echo "$RESPONSE" | grep -q "Access denied for user" && break - sleep 1 + sleep 1 done exit $ret diff --git a/database/mariadb/centos/files/mysql-wait-stop.sh b/database/mariadb/centos/files/mysql-wait-stop.sh index 62bde30ea..0af45fd51 100644 --- a/database/mariadb/centos/files/mysql-wait-stop.sh +++ b/database/mariadb/centos/files/mysql-wait-stop.sh @@ -13,24 +13,24 @@ TIMEOUT=60 SECONDS=0 if ! [ -f "$pidfile" ]; then - exit 0 + exit 0 fi MYSQLPID=`cat "$pidfile" 2>/dev/null` if [ -z "$MYSQLPID" ] ; then - exit 2 + exit 2 fi while /bin/true; do - # Check process still exists - if ! [ -d "/proc/${MYSQLPID}" ] ; then - break - fi - if [ $SECONDS -gt $TIMEOUT ] ; then - ret=3 - break - fi - sleep 1 + # Check process still exists + if ! [ -d "/proc/${MYSQLPID}" ] ; then + break + fi + if [ $SECONDS -gt $TIMEOUT ] ; then + ret=3 + break + fi + sleep 1 done exit $ret diff --git a/filesystem/nfscheck/files/nfscheck-init.sh b/filesystem/nfscheck/files/nfscheck-init.sh index ac2d86920..53b1fccea 100755 --- a/filesystem/nfscheck/files/nfscheck-init.sh +++ b/filesystem/nfscheck/files/nfscheck-init.sh @@ -22,7 +22,7 @@ DESC="nfscheck" DAEMON="/usr/bin/nfscheck" PIDFILE="/var/run/nfscheck.pid" -start() +start() { if [ -e $PIDFILE ]; then PIDDIR=/proc/$(cat $PIDFILE) @@ -47,7 +47,7 @@ start() fi } -stop() +stop() { echo -n "Stopping $DESC..." start-stop-daemon --stop --quiet --pidfile $PIDFILE diff --git a/networking/lldpd/centos/files/i40e-lldp-configure.sh b/networking/lldpd/centos/files/i40e-lldp-configure.sh index 2796c4304..431a405fa 100644 --- a/networking/lldpd/centos/files/i40e-lldp-configure.sh +++ b/networking/lldpd/centos/files/i40e-lldp-configure.sh @@ -28,7 +28,7 @@ LLDP_COMMAND=lldp function log() { local MSG="${PROGNAME}: $1" - logger -p notice "${MSG}" + logger -p notice "${MSG}" } function err() @@ -60,10 +60,10 @@ function configure_device() } function is_debugfs_mounted() { - if grep -qs "${DEBUGFS_PATH}" /proc/mounts; then - return 0 - fi - return 1 + if grep -qs "${DEBUGFS_PATH}" /proc/mounts; then + return 0 + fi + return 1 } function mount_debugfs() { @@ -82,7 +82,7 @@ function scan_devices() if is_debugfs_mounted; then DEBUGFS_MOUNTED="true" - fi + fi if [ ${DEBUGFS_MOUNTED} = "false" ]; then mount_debugfs @@ -120,7 +120,7 @@ function start() function stop() { scan_devices stop - return $? + return $? } function status() @@ -129,20 +129,20 @@ function status() } case "$1" in - start) + start) start ;; - stop) + stop) stop ;; - restart) + restart) stop start ;; - status) + status) status ;; - *) + *) echo "Usage: $0 {start|stop|restart|status}" exit 1 esac diff --git a/networking/mellanox/mlx4-config/files/mlx4-configure.sh b/networking/mellanox/mlx4-config/files/mlx4-configure.sh index 0953f0775..0005ce2e4 100644 --- a/networking/mellanox/mlx4-config/files/mlx4-configure.sh +++ b/networking/mellanox/mlx4-config/files/mlx4-configure.sh @@ -19,7 +19,7 @@ function log() { local MSG="${PROGNAME}: $1" if [ ${DEBUG} -ne 0 ]; then - echo "${MSG}" + echo "${MSG}" fi echo "${MSG}" >> /var/log/mlx4-configure.log } @@ -100,20 +100,20 @@ function status() } case "$1" in - start) + start) start ;; - stop) + stop) stop ;; - restart) + restart) stop start ;; - status) + status) status ;; - *) + *) echo "Usage: $0 {start|stop|restart|status}" exit 1 esac diff --git a/networking/mellanox/mlx4-config/files/mlx4_core_config.sh b/networking/mellanox/mlx4-config/files/mlx4_core_config.sh index 94816bff0..b1c0652d8 100644 --- a/networking/mellanox/mlx4-config/files/mlx4_core_config.sh +++ b/networking/mellanox/mlx4-config/files/mlx4_core_config.sh @@ -1,7 +1,7 @@ #!/bin/bash ################################################################################ # Copyright (c) 2015-2016 Wind River Systems, Inc. -# +# # SPDX-License-Identifier: Apache-2.0 # ################################################################################ diff --git a/tools/collector/scripts/collect_nfv_vim.sh b/tools/collector/scripts/collect_nfv_vim.sh index 4ccd309de..4b9928135 100644 --- a/tools/collector/scripts/collect_nfv_vim.sh +++ b/tools/collector/scripts/collect_nfv_vim.sh @@ -36,7 +36,7 @@ if [ "$nodetype" = "controller" ] ; then DATABASE_DIR=$(awk -F "=" '/database_dir/ {print $2}' /etc/nfv/vim/config.ini) SQLITE_DUMP="/usr/bin/sqlite3 ${DATABASE_DIR}/vim_db_v1 .dump" - + delimiter ${LOGFILE} "dump database" timeout 30 ${SQLITE_DUMP} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} fi diff --git a/tools/collector/scripts/collect_openstack.sh b/tools/collector/scripts/collect_openstack.sh index 743dd5d56..d4e986b62 100755 --- a/tools/collector/scripts/collect_openstack.sh +++ b/tools/collector/scripts/collect_openstack.sh @@ -51,7 +51,7 @@ num_exchanges=$(rabbitmqctl list_exchanges | wc -l); ((num_exchanges-=2)) num_connections=$(rabbitmqctl list_connections | wc -l); ((num_connections-=2)) num_channels=$(rabbitmqctl list_channels | wc -l); ((num_channels-=2)) arr=($(rabbitmqctl list_queues messages consumers memory | \ - awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) + awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]} printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" "queues" "bindings" "exchanges" "connections" "channels" "messages" "consumers" "memory" >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" $num_queues $num_bindings $num_exchanges $num_connections $num_channels $messages $consumers $memory >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} diff --git a/tools/collector/scripts/collect_psqldb.sh b/tools/collector/scripts/collect_psqldb.sh index 410aefa5c..f6613b60c 100755 --- a/tools/collector/scripts/collect_psqldb.sh +++ b/tools/collector/scripts/collect_psqldb.sh @@ -83,7 +83,7 @@ function database_dump() do if [ "$DB" != "keystone" -a "$DB" != "ceilometer" ] ; then echo "${hostname}: Dumping Database ..: ${DB_DIR}/$DB.$DB_EXT" - (cd ${DB_DIR} ; sudo -u postgres pg_dump $DB > $DB.$DB_EXT) + (cd ${DB_DIR} ; sudo -u postgres pg_dump $DB > $DB.$DB_EXT) fi done } @@ -107,7 +107,7 @@ if [ "$nodetype" = "controller" ] ; then pg_size_pretty(pg_database_size(pg_database.datname)) FROM pg_database ORDER BY pg_database_size DESC; - " >> ${LOGFILE} + " >> ${LOGFILE} # Number of postgres connections delimiter ${LOGFILE} "ps -C postgres -o cmd=" diff --git a/tools/engtools/hostdata-collectors/patch-engtools.sh b/tools/engtools/hostdata-collectors/patch-engtools.sh index f64614279..dbb80682e 100755 --- a/tools/engtools/hostdata-collectors/patch-engtools.sh +++ b/tools/engtools/hostdata-collectors/patch-engtools.sh @@ -3,15 +3,15 @@ # http://twiki.wrs.com/PBUeng/Patching if [ -z $MY_WORKSPACE ] || [ -z $MY_REPO ]; then - echo "Some dev environment variables are not set." - echo "Refer to http://wiki.wrs.com/PBUeng/CentOSBuildProcess for instructions." - exit 1 + echo "Some dev environment variables are not set." + echo "Refer to http://wiki.wrs.com/PBUeng/CentOSBuildProcess for instructions." + exit 1 fi ENGTOOLS=$(ls ${MY_WORKSPACE}/std/rpmbuild/RPMS/engtools*noarch.rpm 2>/dev/null) if [ $? -ne 0 ]; then - echo "Engtools RPM has not been built. Please run \"build-pkgs engtools\" first." - exit 1 + echo "Engtools RPM has not been built. Please run \"build-pkgs engtools\" first." + exit 1 fi source ${MY_REPO}/stx/middleware/recipes-common/build-info/release-info.inc diff --git a/tools/engtools/hostdata-collectors/scripts/ceph.sh b/tools/engtools/hostdata-collectors/scripts/ceph.sh index 40014406f..f05904ffc 100644 --- a/tools/engtools/hostdata-collectors/scripts/ceph.sh +++ b/tools/engtools/hostdata-collectors/scripts/ceph.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -16,17 +16,17 @@ OPT_USE_INTERVALS=1 # Print key ceph statistics function print_ceph() { - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME - cmd='ceph -s' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='ceph -s' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='ceph osd tree' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='ceph osd tree' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='ceph df detail' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='ceph df detail' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} } #------------------------------------------------------------------------------- @@ -49,8 +49,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_ceph - sleep ${INTERVAL_SEC} + print_ceph + sleep ${INTERVAL_SEC} done print_ceph LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh b/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh index e1f16239d..8fc7b8b1e 100644 --- a/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -ne 0 ]; then - ERRLOG "Require sudo/root access." - exit 1 + ERRLOG "Require sudo/root access." + exit 1 fi declare -a TOOLS @@ -34,23 +34,23 @@ LOG "kill processes brute force" pids=( $(pidof -x /usr/local/bin/collect-engtools.sh) ) if [ ${#pids[@]} -ne 0 ] then - LOG "killing: ${pids[@]}" - for pid in ${pids[@]} - do + LOG "killing: ${pids[@]}" + for pid in ${pids[@]} + do LOG "kill: [ ${pid} ] " pkill -KILL -P ${pid} kill -9 ${pid} - done - pkill -KILL iostat - pkill -KILL top + done + pkill -KILL iostat + pkill -KILL top else - LOG "no pids found" + LOG "no pids found" fi LOG "remove pidfiles" for TOOL in "${TOOLS[@]}" do - rm -f -v /var/run/${TOOL}.pid + rm -f -v /var/run/${TOOL}.pid done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh b/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh index 7c1887a13..5e3c06430 100644 --- a/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh @@ -50,26 +50,26 @@ declare tgt_used_bytes # do_parallel_commands - launch parallel tools with separate output files function do_parallel_commands() { - parallel_outfiles=() - for elem in "${tlist[@]}" - do - tool=""; period=""; repeat=""; interval="" - my_hash="elem[*]" - local ${!my_hash} - if [ ! -z "${name}" ]; then - fname="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${name}" - parallel_outfiles+=( $fname ) - LOG "collecting ${tool}, ${interval} second intervals, to: ${fname}" - if [ ! -z "${period}" ]; then - ${STDBUF} ${tool} -p ${period} -i ${interval} > ${fname} 2>/dev/null & - elif [ ! -z "${repeat}" ]; then - ${STDBUF} ${tool} --repeat=${repeat} --delay=${interval} > ${fname} 2>/dev/null & - fi - else - # run without file output (eg., ticker) - ${STDBUF} ${tool} -p ${period} -i ${interval} 2>/dev/null & - fi - done + parallel_outfiles=() + for elem in "${tlist[@]}" + do + tool=""; period=""; repeat=""; interval="" + my_hash="elem[*]" + local ${!my_hash} + if [ ! -z "${name}" ]; then + fname="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${name}" + parallel_outfiles+=( $fname ) + LOG "collecting ${tool}, ${interval} second intervals, to: ${fname}" + if [ ! -z "${period}" ]; then + ${STDBUF} ${tool} -p ${period} -i ${interval} > ${fname} 2>/dev/null & + elif [ ! -z "${repeat}" ]; then + ${STDBUF} ${tool} --repeat=${repeat} --delay=${interval} > ${fname} 2>/dev/null & + fi + else + # run without file output (eg., ticker) + ${STDBUF} ${tool} -p ${period} -i ${interval} 2>/dev/null & + fi + done } # get_current_avail_usage() - get output destination file-system usage and @@ -77,10 +77,10 @@ function do_parallel_commands() # - updates: df_size_bytes, df_avail_bytes, du_used_bytes function get_current_avail_usage() { - local -a df_arr_bytes=( $(df -P --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==2 {print $2, $4}') ) - df_size_bytes=${df_arr_bytes[0]} - df_avail_bytes=${df_arr_bytes[1]} - du_used_bytes=$(du --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==1 {print $1}') + local -a df_arr_bytes=( $(df -P --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==2 {print $2, $4}') ) + df_size_bytes=${df_arr_bytes[0]} + df_avail_bytes=${df_arr_bytes[1]} + du_used_bytes=$(du --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==1 {print $1}') } # purge_oldest_files() - remove oldest files based on file-system available space, @@ -88,57 +88,57 @@ function get_current_avail_usage() function purge_oldest_files() { # get current file-system usage - get_current_avail_usage - msg=$(printf "avail %d MB, headroom %d MB; used %d MB, max %d MB" \ + get_current_avail_usage + msg=$(printf "avail %d MB, headroom %d MB; used %d MB, max %d MB" \ $[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] \ $[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024]) - LOG "usage: ${msg}" + LOG "usage: ${msg}" - if [[ $df_avail_bytes -lt $tgt_avail_bytes ]] || \ - [[ $du_used_bytes -gt $tgt_used_bytes ]]; then - # wait for compression to complete - wait + if [[ $df_avail_bytes -lt $tgt_avail_bytes ]] || \ + [[ $du_used_bytes -gt $tgt_used_bytes ]]; then + # wait for compression to complete + wait - get_current_avail_usage - if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then - msg=$(printf "purge: avail %d MB < target %d MB" \ - $[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] ) - LOG "purge: ${msg}" + get_current_avail_usage + if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then + msg=$(printf "purge: avail %d MB < target %d MB" \ + $[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] ) + LOG "purge: ${msg}" + fi + if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then + msg=$(printf "purge: used %d MB > target %d MB" \ + $[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024] ) + LOG "purge: ${msg}" + fi + else + return fi - if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then - msg=$(printf "purge: used %d MB > target %d MB" \ - $[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024] ) - LOG "purge: ${msg}" - fi - else - return - fi # remove files in oldest time sorted order until we meet usage targets, # incrementally updating usage as we remve files - for file in $( ls -rt ${TOOL_DEST_DIR}/${HOSTNAME}_* 2>/dev/null ) - do - if [[ $df_avail_bytes -ge $tgt_avail_bytes ]] && \ - [[ $du_used_bytes -le $tgt_used_bytes ]]; then - break - fi + for file in $( ls -rt ${TOOL_DEST_DIR}/${HOSTNAME}_* 2>/dev/null ) + do + if [[ $df_avail_bytes -ge $tgt_avail_bytes ]] && \ + [[ $du_used_bytes -le $tgt_used_bytes ]]; then + break + fi - if [ ${OPT_DEBUG} -eq 1 ]; then - msg="purge: file=$file" - if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then - msg="${msg}, < AVAIL" - fi - if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then - msg="${msg}, > MAXUSAGE" - fi - LOG "${msg}" - fi + if [ ${OPT_DEBUG} -eq 1 ]; then + msg="purge: file=$file" + if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then + msg="${msg}, < AVAIL" + fi + if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then + msg="${msg}, > MAXUSAGE" + fi + LOG "${msg}" + fi - sz_bytes=$(stat --printf="%s" $file) - ((df_avail_bytes += sz_bytes)) - ((du_used_bytes -= sz_bytes)) - rm -fv ${file} - done + sz_bytes=$(stat --printf="%s" $file) + ((df_avail_bytes += sz_bytes)) + ((du_used_bytes -= sz_bytes)) + rm -fv ${file} + done } #------------------------------------------------------------------------------- @@ -160,23 +160,23 @@ CPULIST="" # Affine tools to NOVA pinned cores (i.e., non-cpu 0) # - remove interference with cpu 0 if [ "${AFFINE_PINNED}" -eq 1 ]; then - NOVA_CONF=/etc/nova/compute_extend.conf - if [ -f "${NOVA_CONF}" ]; then + NOVA_CONF=/etc/nova/compute_extend.conf + if [ -f "${NOVA_CONF}" ]; then source "${NOVA_CONF}" CPULIST=${compute_pinned_cpulist} - else + else CPULIST="" - fi + fi fi set_affinity ${CPULIST} # Define output directory if [[ "${HOSTNAME}" =~ "controller-" ]]; then - TOOL_DEST_DIR=/scratch/syseng_data/${HOSTNAME} + TOOL_DEST_DIR=/scratch/syseng_data/${HOSTNAME} elif [[ "${HOSTNAME}" =~ "compute-" ]]; then - TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} + TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} else - TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} + TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} fi mkdir -p ${TOOL_DEST_DIR} @@ -186,7 +186,7 @@ DAEMON_OUT="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${TOOLNAME}.log" # Redirect stdout and append to log if not connected to TTY if test ! -t 1 ; then - exec 1>> ${DAEMON_OUT} + exec 1>> ${DAEMON_OUT} fi # Get current availability and usage @@ -199,9 +199,9 @@ tgt_used_bytes=$[$PURGE_MAXUSAGE_MB*1024*1024] # Set granularity based on duration if [ $PERIOD_MIN -le 30 ]; then - GRAN_MIN=5 + GRAN_MIN=5 else - GRAN_MIN=60 + GRAN_MIN=60 fi # Adjust repeats and intervals based on GRAN_MIN granularity @@ -209,7 +209,7 @@ PERIOD_MIN=$[($PERIOD_MIN+(GRAN_MIN-1))/GRAN_MIN*GRAN_MIN] ((REPEATS = PERIOD_MIN/GRAN_MIN)) GRAN_MIN_IN_SEC=$[$GRAN_MIN*60] if [ ${INTERVAL_SEC} -gt ${GRAN_MIN_IN_SEC} ]; then - INTERVAL_SEC=${GRAN_MIN_IN_SEC} + INTERVAL_SEC=${GRAN_MIN_IN_SEC} fi # Define tools and options @@ -231,7 +231,7 @@ if [ ${OPT_SOAK} -eq 1 ]; then ceph_interval=${DUR_15MIN_IN_SEC} diskstats_interval=${DUR_15MIN_IN_SEC} memstats_interval=${DUR_15MIN_IN_SEC} - filestats_interval=${DUR_15MIN_IN_SEC} + filestats_interval=${DUR_15MIN_IN_SEC} elif [ ${OPT_SOAK} -eq 2 ]; then # Assume much shorter collection (eg, < hours) schedtop_interval=2 # i.e., 2 second interval @@ -243,7 +243,7 @@ elif [ ${OPT_SOAK} -eq 2 ]; then ceph_interval=${DUR_5MIN_IN_SEC} diskstats_interval=${DUR_5MIN_IN_SEC} memstats_interval=${DUR_5MIN_IN_SEC} - filestats_interval=${DUR_5MIN_IN_SEC} + filestats_interval=${DUR_5MIN_IN_SEC} else # Assume shorter collection (eg, < a few hours) schedtop_interval=5 # i.e., 5 second interval @@ -255,7 +255,7 @@ else ceph_interval=${DUR_5MIN_IN_SEC} diskstats_interval=${DUR_5MIN_IN_SEC} memstats_interval=${DUR_5MIN_IN_SEC} - filestats_interval=${DUR_5MIN_IN_SEC} + filestats_interval=${DUR_5MIN_IN_SEC} fi schedtop_repeat=$[ $PERIOD_MIN * 60 / $schedtop_interval ] occtop_repeat=$[ $PERIOD_MIN * 60 / $occtop_interval ] @@ -274,29 +274,29 @@ LBINDIR=/usr/local/bin declare -a tlist if [[ ${ENABLE_STATIC_COLLECTION} == "Y" ]] || [[ ${ENABLE_STATIC_COLLECTION} == "y" ]]; then - tlist+=( "tool=${LBINDIR}/top.sh name=top period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) - tlist+=( "tool=${LBINDIR}/iostat.sh name=iostat period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) - tlist+=( "tool=${LBINDIR}/netstats.sh name=netstats period=${PERIOD_MIN} interval=${netstats_interval}" ) - tlist+=( "tool=${BINDIR}/occtop name=occtop repeat=${occtop_repeat} interval=${occtop_interval}" ) - tlist+=( "tool=${BINDIR}/memtop name=memtop repeat=${memtop_repeat} interval=${memtop_interval}" ) - tlist+=( "tool=${BINDIR}/schedtop name=schedtop repeat=${schedtop_repeat} interval=${schedtop_interval}" ) - tlist+=( "tool=${LBINDIR}/diskstats.sh name=diskstats period=${PERIOD_MIN} interval=${diskstats_interval}" ) - tlist+=( "tool=${LBINDIR}/memstats.sh name=memstats period=${PERIOD_MIN} interval=${memstats_interval}" ) - tlist+=( "tool=${LBINDIR}/filestats.sh name=filestats period=${PERIOD_MIN} interval=${filestats_interval}" ) - if [[ "${HOSTNAME}" =~ "controller-" ]]; then - tlist+=( "tool=${LBINDIR}/ceph.sh name=ceph period=${PERIOD_MIN} interval=${ceph_interval}" ) - tlist+=( "tool=${LBINDIR}/postgres.sh name=postgres period=${PERIOD_MIN} interval=${postgres_interval}" ) - tlist+=( "tool=${LBINDIR}/rabbitmq.sh name=rabbitmq period=${PERIOD_MIN} interval=${rabbitmq_interval}" ) - elif [[ "${HOSTNAME}" =~ "compute-" ]]; then - tlist+=( "tool=${LBINDIR}/vswitch.sh name=vswitch period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) - fi + tlist+=( "tool=${LBINDIR}/top.sh name=top period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + tlist+=( "tool=${LBINDIR}/iostat.sh name=iostat period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + tlist+=( "tool=${LBINDIR}/netstats.sh name=netstats period=${PERIOD_MIN} interval=${netstats_interval}" ) + tlist+=( "tool=${BINDIR}/occtop name=occtop repeat=${occtop_repeat} interval=${occtop_interval}" ) + tlist+=( "tool=${BINDIR}/memtop name=memtop repeat=${memtop_repeat} interval=${memtop_interval}" ) + tlist+=( "tool=${BINDIR}/schedtop name=schedtop repeat=${schedtop_repeat} interval=${schedtop_interval}" ) + tlist+=( "tool=${LBINDIR}/diskstats.sh name=diskstats period=${PERIOD_MIN} interval=${diskstats_interval}" ) + tlist+=( "tool=${LBINDIR}/memstats.sh name=memstats period=${PERIOD_MIN} interval=${memstats_interval}" ) + tlist+=( "tool=${LBINDIR}/filestats.sh name=filestats period=${PERIOD_MIN} interval=${filestats_interval}" ) + if [[ "${HOSTNAME}" =~ "controller-" ]]; then + tlist+=( "tool=${LBINDIR}/ceph.sh name=ceph period=${PERIOD_MIN} interval=${ceph_interval}" ) + tlist+=( "tool=${LBINDIR}/postgres.sh name=postgres period=${PERIOD_MIN} interval=${postgres_interval}" ) + tlist+=( "tool=${LBINDIR}/rabbitmq.sh name=rabbitmq period=${PERIOD_MIN} interval=${rabbitmq_interval}" ) + elif [[ "${HOSTNAME}" =~ "compute-" ]]; then + tlist+=( "tool=${LBINDIR}/vswitch.sh name=vswitch period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + fi # ticker - shows progress on the screen - tlist+=( "tool=${LBINDIR}/ticker.sh name= period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + tlist+=( "tool=${LBINDIR}/ticker.sh name= period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) fi if [[ ${ENABLE_LIVE_STREAM} == "Y" ]] || [[ ${ENABLE_LIVE_STREAM} == "y" ]]; then - ${TOOLBIN}/live_stream.py & + ${TOOLBIN}/live_stream.py & fi #------------------------------------------------------------------------------- @@ -306,32 +306,32 @@ OPT_DEBUG=0 REP=0 if [ ${#tlist[@]} -ne 0 ]; then - # Static stats collection is turned on - while [[ ${TOOL_USR1_SIGNAL} -eq 0 ]] && + # Static stats collection is turned on + while [[ ${TOOL_USR1_SIGNAL} -eq 0 ]] && [[ ${OPT_FOREVER} -eq 1 || ${REP} -lt ${REPEATS} ]] - do - # increment loop counter - ((REP++)) + do + # increment loop counter + ((REP++)) - # purge oldest files - purge_oldest_files + # purge oldest files + purge_oldest_files - # define filename timestamp - timestamp=$( date +"%Y-%0m-%0e_%H%M" ) + # define filename timestamp + timestamp=$( date +"%Y-%0m-%0e_%H%M" ) - # collect tools in parallel to separate output files - LOG "collecting ${TOOLNAME} at ${timestamp} for ${PERIOD_MIN} mins, repeat=${REP}" - do_parallel_commands - wait + # collect tools in parallel to separate output files + LOG "collecting ${TOOLNAME} at ${timestamp} for ${PERIOD_MIN} mins, repeat=${REP}" + do_parallel_commands + wait + + # Compress latest increment + LOG "compressing: ${parallel_outfiles[@]}" + ${CMD_IDLE} bzip2 -q -f ${parallel_outfiles[@]} 2>/dev/null & + done - # Compress latest increment - LOG "compressing: ${parallel_outfiles[@]}" - ${CMD_IDLE} bzip2 -q -f ${parallel_outfiles[@]} 2>/dev/null & - done - # Wait for the compression to complete - wait - tools_cleanup 0 + wait + tools_cleanup 0 fi # Should wait here in case live stats streaming is turned on. diff --git a/tools/engtools/hostdata-collectors/scripts/diskstats.sh b/tools/engtools/hostdata-collectors/scripts/diskstats.sh index 376dbf185..d00d777ba 100644 --- a/tools/engtools/hostdata-collectors/scripts/diskstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/diskstats.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -16,76 +16,76 @@ OPT_USE_INTERVALS=1 # Print disk summary function print_disk() { - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME # NOTES: # --total (grand-total) is a new option, but don't necessarily want to add tmpfs - # or dummy filesystems. + # or dummy filesystems. # - use -H to print in SI (eg, GB, vs GiB) # - can use -a to print all filesystems including dummy filesystems, but then # there can be double-counting: - print_separator - cmd='df -h -H -T --local -t ext2 -t ext3 -t ext4 -t xfs --total' - ${ECHO} "Disk space usage ext2,ext3,ext4,xfs,tmpfs (SI):" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='df -h -H -T --local -t ext2 -t ext3 -t ext4 -t xfs --total' + ${ECHO} "Disk space usage ext2,ext3,ext4,xfs,tmpfs (SI):" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='df -h -H -T --local -i -t ext2 -t ext3 -t ext4 -t xfs --total' - ${ECHO} "Disk inodes usage ext2,ext3,ext4,xfs,tmpfs (SI):" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='df -h -H -T --local -i -t ext2 -t ext3 -t ext4 -t xfs --total' + ${ECHO} "Disk inodes usage ext2,ext3,ext4,xfs,tmpfs (SI):" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='drbd-overview' - ${ECHO} "drbd disk usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='drbd-overview' + ${ECHO} "drbd disk usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='lvs' - ${ECHO} "logical volumes usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='lvs' + ${ECHO} "logical volumes usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='pvs' - ${ECHO} "physical volumes usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='pvs' + ${ECHO} "physical volumes usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='vgs' - ${ECHO} "volume groups usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='vgs' + ${ECHO} "volume groups usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} } # Print disk static summary function print_disk_static() { - print_separator - cmd='cat /proc/scsi/scsi' - ${ECHO} "Attached devices: ${cmd}" - ${cmd} - ${ECHO} + print_separator + cmd='cat /proc/scsi/scsi' + ${ECHO} "Attached devices: ${cmd}" + ${cmd} + ${ECHO} # fdisk - requires sudo/root - print_separator - cmd='fdisk -l' - if [ $UID -eq 0 ]; then - ${ECHO} "List disk devices: ${cmd}" - ${cmd} - else - WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" - fi - ${ECHO} + print_separator + cmd='fdisk -l' + if [ $UID -eq 0 ]; then + ${ECHO} "List disk devices: ${cmd}" + ${cmd} + else + WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" + fi + ${ECHO} # parted - requires sudo/root - print_separator - cmd='parted -l' - if [ $UID -eq 0 ]; then - ${ECHO} "List disk devices: ${cmd}" - ${cmd} - else - WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" - fi - ${ECHO} + print_separator + cmd='parted -l' + if [ $UID -eq 0 ]; then + ${ECHO} "List disk devices: ${cmd}" + ${cmd} + else + WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" + fi + ${ECHO} } #------------------------------------------------------------------------------- @@ -111,8 +111,8 @@ print_disk_static for ((rep=1; rep <= REPEATS ; rep++)) do - print_disk - sleep ${INTERVAL_SEC} + print_disk + sleep ${INTERVAL_SEC} done print_disk LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/engtools_util.sh b/tools/engtools/hostdata-collectors/scripts/engtools_util.sh index 311ccd2bf..4ac66a9ae 100644 --- a/tools/engtools/hostdata-collectors/scripts/engtools_util.sh +++ b/tools/engtools/hostdata-collectors/scripts/engtools_util.sh @@ -7,7 +7,7 @@ TOOL_USR1_SIGNAL=0 TOOL_USR2_SIGNAL=0 TOOL_TTY=0 if tty 1>/dev/null ; then - TOOL_TTY=1 + TOOL_TTY=1 fi # [ JGAULD : SHOULD RENAME TO TOOL_X ] @@ -19,145 +19,145 @@ CPULIST=0 # Include lsb functions if [ -d /lib/lsb ]; then -. /lib/lsb/init-functions + . /lib/lsb/init-functions else -. /etc/init.d/functions + . /etc/init.d/functions fi # Lightweight replacement for pidofproc -p function check_pidfile () { - local pidfile pid + local pidfile pid - OPTIND=1 - while getopts p: opt ; do + OPTIND=1 + while getopts p: opt ; do case "$opt" in - p) + p) pidfile="$OPTARG" ;; esac - done - shift $(($OPTIND - 1)) + done + shift $(($OPTIND - 1)) - read pid < "${pidfile}" - if [ -n "${pid:-}" ]; then - if $(kill -0 "${pid:-}" 2> /dev/null); then - echo "$pid" - return 0 - elif ps "${pid:-}" >/dev/null 2>&1; then - echo "$pid" - return 0 # program is running, but not owned by this user - else - return 1 # program is dead and /var/run pid file exists + read pid < "${pidfile}" + if [ -n "${pid:-}" ]; then + if $(kill -0 "${pid:-}" 2> /dev/null); then + echo "$pid" + return 0 + elif ps "${pid:-}" >/dev/null 2>&1; then + echo "$pid" + return 0 # program is running, but not owned by this user + else + return 1 # program is dead and /var/run pid file exists + fi fi - fi } # tools_init - initialize tool resources function tools_init () { - local rc=0 - local error=0 - TOOLNAME=$(basename $0) + local rc=0 + local error=0 + TOOLNAME=$(basename $0) # Check for sufficient priviledges - if [ $UID -ne 0 ]; then - ERRLOG "${NAME} requires sudo/root access." - return 1 - fi + if [ $UID -ne 0 ]; then + ERRLOG "${NAME} requires sudo/root access." + return 1 + fi # Check for essential binaries - ECHO=$(which echo 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ECHO=echo # use bash built-in echo - ${ECHO} "FATAL, 'echo' not found, rc=$rc"; - error=$rc - fi - DATE=$(which date 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ${ECHO} "FATAL, 'date' not found, rc=$rc"; - error=$rc - fi + ECHO=$(which echo 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ECHO=echo # use bash built-in echo + ${ECHO} "FATAL, 'echo' not found, rc=$rc"; + error=$rc + fi + DATE=$(which date 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ${ECHO} "FATAL, 'date' not found, rc=$rc"; + error=$rc + fi # Check for standard linux binaries, at least can use LOG functions now # - these are used in tools_header - CAT=$(which cat 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'cat' not found, rc=$rc"; - error=$rc - fi + CAT=$(which cat 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'cat' not found, rc=$rc"; + error=$rc + fi - ARCH=$(which arch 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'arch' not found, rc=$rc"; - error=$rc - fi + ARCH=$(which arch 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'arch' not found, rc=$rc"; + error=$rc + fi - SED=$(which sed 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'sed' not found, rc=$rc"; - error=$rc - fi + SED=$(which sed 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'sed' not found, rc=$rc"; + error=$rc + fi - GREP=$(which grep 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'grep' not found, rc=$rc"; - error=$rc - fi + GREP=$(which grep 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'grep' not found, rc=$rc"; + error=$rc + fi - WC=$(which wc 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'wc' not found, rc=$rc"; - error=$rc - fi + WC=$(which wc 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'wc' not found, rc=$rc"; + error=$rc + fi - UNAME=$(which uname 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'uname' not found, rc=$rc"; - error=$rc - fi + UNAME=$(which uname 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'uname' not found, rc=$rc"; + error=$rc + fi - SORT=$(which sort 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'sort' not found, rc=$rc"; - error=$rc - fi + SORT=$(which sort 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'sort' not found, rc=$rc"; + error=$rc + fi - TR=$(which tr 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'tr' not found, rc=$rc"; - error=$rc - fi + TR=$(which tr 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'tr' not found, rc=$rc"; + error=$rc + fi - AWK=$(which awk 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'awk' not found, rc=$rc"; - error=$rc - fi + AWK=$(which awk 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'awk' not found, rc=$rc"; + error=$rc + fi - PKILL=$(which pkill 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'pkill' not found, rc=$rc"; - error=$rc - fi + PKILL=$(which pkill 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'pkill' not found, rc=$rc"; + error=$rc + fi - LS=$(which ls 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'ls' not found, rc=$rc"; - error=$rc - fi + LS=$(which ls 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'ls' not found, rc=$rc"; + error=$rc + fi # The following block is needed for LSB systems such as Windriver Linux. # The utility is not available on CentOS so comment it out. @@ -170,91 +170,91 @@ function tools_init () # Let parent program decide what to do with the errors, # give ominous warning - if [ $error -eq 1 ]; then - WARNLOG "possibly cannot continue, missing linux binaries" - fi + if [ $error -eq 1 ]; then + WARNLOG "possibly cannot continue, missing linux binaries" + fi # Check if tool was previously running - if [ -e ${PIDFILE} ]; then + if [ -e ${PIDFILE} ]; then # [ JGAULD - remove pidofproc() / LSB compatibility issue ] if check_pidfile -p "${PIDFILE}" >/dev/null; then - ERRLOG "${PIDFILE} exists and ${TOOLNAME} is running" - return 1 + ERRLOG "${PIDFILE} exists and ${TOOLNAME} is running" + return 1 else # remove pid file - WARNLOG "${PIDFILE} exists but ${TOOLNAME} is not running; cleaning up" - rm -f ${PIDFILE} + WARNLOG "${PIDFILE} exists but ${TOOLNAME} is not running; cleaning up" + rm -f ${PIDFILE} + fi fi - fi # Create pid file - echo $$ > ${PIDFILE} + echo $$ > ${PIDFILE} # Setup trap handler - these signals trigger child shutdown and cleanup - trap tools_exit_handler INT HUP TERM EXIT - trap tools_usr1_handler USR1 - trap tools_usr2_handler USR2 + trap tools_exit_handler INT HUP TERM EXIT + trap tools_usr1_handler USR1 + trap tools_usr2_handler USR2 - return ${rc} + return ${rc} } # tools_cleanup() - terminate child processes function tools_cleanup() { # restore signal handling to default behaviour - trap - INT HUP TERM EXIT - trap - USR1 USR2 + trap - INT HUP TERM EXIT + trap - USR1 USR2 - local VERBOSE_OPT='' - if [ "$1" -ne "0" ]; then - LOG "cleanup invoked with code: $1" - if [ ${TOOL_DEBUG} -ne 0 ]; then - VERBOSE_OPT='-v' + local VERBOSE_OPT='' + if [ "$1" -ne "0" ]; then + LOG "cleanup invoked with code: $1" + if [ ${TOOL_DEBUG} -ne 0 ]; then + VERBOSE_OPT='-v' + fi fi - fi # stop all processes launched from this process - pkill -TERM -P $$ - if [ "$1" -ne "0" ]; then - sleep 1 - fi + pkill -TERM -P $$ + if [ "$1" -ne "0" ]; then + sleep 1 + fi # OK, if the above didn't work, use force - pkill -KILL -P $$ + pkill -KILL -P $$ # remove pid file - if [ -e ${PIDFILE} ]; then - rm -f ${VERBOSE_OPT} ${PIDFILE} - fi - exit $1 + if [ -e ${PIDFILE} ]; then + rm -f ${VERBOSE_OPT} ${PIDFILE} + fi + exit $1 } # tools_exit_handler() - exit handler routine function tools_exit_handler() { - TOOL_EXIT_SIGNAL=1 - tools_cleanup 128 + TOOL_EXIT_SIGNAL=1 + tools_cleanup 128 } # tools_usr1_handler() - USR1 handler routine function tools_usr1_handler() { - TOOL_USR1_SIGNAL=1 - LOG "caught USR1" + TOOL_USR1_SIGNAL=1 + LOG "caught USR1" } # tools_usr2_handler() - USR2 handler routine function tools_usr2_handler() { - TOOL_USR2_SIGNAL=1 - LOG "caught USR1" + TOOL_USR2_SIGNAL=1 + LOG "caught USR1" } # LOG(), WARNLOG(), ERRLOG() - simple print log functions (not logger) function LOG () { - local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) - echo "${tstamp_H} ${HOSTNAME} $0($$): $@"; + local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) + echo "${tstamp_H} ${HOSTNAME} $0($$): $@"; } function LOG_NOCR () { - local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) - echo -n "${tstamp_H} ${HOSTNAME} $0($$): $@"; + local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) + echo -n "${tstamp_H} ${HOSTNAME} $0($$): $@"; } function WARNLOG () { LOG "WARN $@"; } function ERRLOG () { LOG "ERROR $@"; } @@ -262,76 +262,76 @@ function ERRLOG () { LOG "ERROR $@"; } # TOOL_HIRES_TIME() - easily parsed date/timestamp and hi-resolution uptime function TOOL_HIRES_TIME() { - echo "time: " $( ${DATE} +"%a %F %H:%M:%S.%N %Z %z" ) "uptime: " $( cat /proc/uptime ) + echo "time: " $( ${DATE} +"%a %F %H:%M:%S.%N %Z %z" ) "uptime: " $( cat /proc/uptime ) } # set_affinity() - set affinity for current script if a a CPULIST is defined function set_affinity() { - local CPULIST=$1 - if [ -z "${CPULIST}" ]; then - return - fi + local CPULIST=$1 + if [ -z "${CPULIST}" ]; then + return + fi # Set cpu affinity for current program - local TASKSET=$(which taskset 2>/dev/null) - if [ -x "${TASKSET}" ]; then - ${TASKSET} -pc ${CPULIST} $$ 2>/dev/null - fi + local TASKSET=$(which taskset 2>/dev/null) + if [ -x "${TASKSET}" ]; then + ${TASKSET} -pc ${CPULIST} $$ 2>/dev/null + fi } # cmd_idle_priority() - command to set nice + ionice function cmd_idle_priority() { - local NICE="" - local IONICE="" + local NICE="" + local IONICE="" - NICE=$( which nice 2>/dev/null ) - if [ $? -eq 0 ]; then - NICE="${NICE} -n 19" - else - NICE="" - fi - IONICE=$( which ionice 2>/dev/null ) - if [ $? -eq 0 ]; then - IONICE="${IONICE} -c 3" - else - IONICE="" - fi - echo "${NICE} ${IONICE}" + NICE=$( which nice 2>/dev/null ) + if [ $? -eq 0 ]; then + NICE="${NICE} -n 19" + else + NICE="" + fi + IONICE=$( which ionice 2>/dev/null ) + if [ $? -eq 0 ]; then + IONICE="${IONICE} -c 3" + else + IONICE="" + fi + echo "${NICE} ${IONICE}" } # print_separator() - print a horizontal separation line '\u002d' is '-' function print_separator () { - printf '\u002d%.s' {1..80} - printf '\n' + printf '\u002d%.s' {1..80} + printf '\n' } # tools_header() - print out common GenWare tools header function tools_header() { - local TOOLNAME=$(basename $0) + local TOOLNAME=$(basename $0) # Get timestamp #local tstamp=$( date +"%Y-%0m-%0e %H:%M:%S" 2>/dev/null ) - local tstamp=$( date --rfc-3339=ns | cut -c1-23 2>/dev/null ) + local tstamp=$( date --rfc-3339=ns | cut -c1-23 2>/dev/null ) # Linux Generic - local UPTIME=/proc/uptime + local UPTIME=/proc/uptime # Get number of online cpus - local CPUINFO=/proc/cpuinfo - local online_cpus=$( cat ${CPUINFO} | grep -i ^processor | wc -l 2>/dev/null ) + local CPUINFO=/proc/cpuinfo + local online_cpus=$( cat ${CPUINFO} | grep -i ^processor | wc -l 2>/dev/null ) # Get load average, run-queue size, and number of threads - local LOADAVG=/proc/loadavg - local LDAVG=( `cat ${LOADAVG} | sed -e 's#[/]# #g' 2>/dev/null` ) + local LOADAVG=/proc/loadavg + local LDAVG=( `cat ${LOADAVG} | sed -e 's#[/]# #g' 2>/dev/null` ) # Get current architecture - local arch=$( uname -m ) + local arch=$( uname -m ) # Determine processor name (there are many different formats... *sigh* ) # - build up info from multiple lines - local processor='unk' - local NAME=$( cat ${CPUINFO} | grep \ + local processor='unk' + local NAME=$( cat ${CPUINFO} | grep \ -e '^cpu\W\W:' \ -e ^'cpu model' \ -e ^'model name' \ @@ -340,53 +340,53 @@ function tools_header() { -e ^[Mm]achine | \ sort -u | awk 'BEGIN{FS=":";} {print $2;}' | \ tr '\n' ' ' | tr -s [:blank:] 2>/dev/null ) - if [ ! -z "${NAME}" ]; then - processor=${NAME} - fi + if [ ! -z "${NAME}" ]; then + processor=${NAME} + fi # Determine processor speed (abort grep after first match) - local speed='unk' - local BOGO=$( cat ${CPUINFO} | grep -m1 -e ^BogoMIPS -e ^bogomips | \ + local speed='unk' + local BOGO=$( cat ${CPUINFO} | grep -m1 -e ^BogoMIPS -e ^bogomips | \ awk 'BEGIN{FS=":";} {printf "%.1f", $2;}' 2>/dev/null ) - local MHZ=$( cat ${CPUINFO} | grep -m1 -e ^'cpu MHz' -e ^clock | \ + local MHZ=$( cat ${CPUINFO} | grep -m1 -e ^'cpu MHz' -e ^clock | \ awk 'BEGIN{FS=":";} {printf "%.1f", $2;}' 2>/dev/null ) - local MHZ2=$( cat ${CPUINFO} | grep -m1 -e ^Cpu0ClkTck -e ^'cycle frequency' | \ + local MHZ2=$( cat ${CPUINFO} | grep -m1 -e ^Cpu0ClkTck -e ^'cycle frequency' | \ awk 'BEGIN{FS=":";} {printf "%.1f", $2/1.0E6;}' 2>/dev/null ) - if [ ! -z "${MHZ}" ]; then - speed=${MHZ} - elif [ ! -z "${MHZ2}" ]; then - speed=${MHZ2} - elif [ ! -z ${BOGO} ]; then - speed=${BOGO} - fi + if [ ! -z "${MHZ}" ]; then + speed=${MHZ} + elif [ ! -z "${MHZ2}" ]; then + speed=${MHZ2} + elif [ ! -z ${BOGO} ]; then + speed=${BOGO} + fi # Determine OS and kernel version - local os_name=$( uname -s 2>/dev/null ) - local os_release=$( uname -r 2>/dev/null ) + local os_name=$( uname -s 2>/dev/null ) + local os_release=$( uname -r 2>/dev/null ) - declare -a arr + declare -a arr - local dist_id="" + local dist_id="" # Determine OS distribution ID - if [ lsb_pres == "yes" ]; then - arr=( $( lsb_release -i 2>/dev/null ) ) - dist_id=${arr[2]} - else - local dist_id=$(cat /etc/centos-release | awk '{print $1}' 2>/dev/null) - fi - - local dist_rel="" - if [ lsb_pres == "yes" ]; then + if [ lsb_pres == "yes" ]; then + arr=( $( lsb_release -i 2>/dev/null ) ) + dist_id=${arr[2]} + else + local dist_id=$(cat /etc/centos-release | awk '{print $1}' 2>/dev/null) + fi + + local dist_rel="" + if [ lsb_pres == "yes" ]; then # Determine OS distribution release - arr=( $( cat /proc/version | awk '{print $3}' 2>/dev/null ) ) - local dist_rel=${arr[1]} - else - local dist_rel=$(cat /etc/centos-release | awk '{print $4}' 2>/dev/null) - fi + arr=( $( cat /proc/version | awk '{print $3}' 2>/dev/null ) ) + local dist_rel=${arr[1]} + else + local dist_rel=$(cat /etc/centos-release | awk '{print $4}' 2>/dev/null) + fi # Print generic header - echo "${TOOLNAME} -- ${tstamp} load average:${LDAVG[0]}, ${LDAVG[1]}, ${LDAVG[2]} runq:${LDAVG[3]} nproc:${LDAVG[4]}" - echo " host:${HOSTNAME} Distribution:${dist_id} ${dist_rel} ${os_name} ${os_release}" - echo " arch:${arch} processor:${processor} speed:${speed} MHz CPUs:${online_cpus}" + echo "${TOOLNAME} -- ${tstamp} load average:${LDAVG[0]}, ${LDAVG[1]}, ${LDAVG[2]} runq:${LDAVG[3]} nproc:${LDAVG[4]}" + echo " host:${HOSTNAME} Distribution:${dist_id} ${dist_rel} ${os_name} ${os_release}" + echo " arch:${arch} processor:${processor} speed:${speed} MHz CPUs:${online_cpus}" } @@ -394,85 +394,85 @@ function tools_header() { # tools_usage() - show generic tools tool usage function tools_usage() { - if [ ${OPT_USE_INTERVALS} -eq 1 ]; then - echo "usage: ${TOOLNAME} [-f] [-p ] [-i ] [-c ] [-h]" - else - echo "Usage: ${TOOLNAME} [-f] [-p ] [-c ] [-h]" - fi + if [ ${OPT_USE_INTERVALS} -eq 1 ]; then + echo "usage: ${TOOLNAME} [-f] [-p ] [-i ] [-c ] [-h]" + else + echo "Usage: ${TOOLNAME} [-f] [-p ] [-c ] [-h]" + fi } # tools_print_help() - print generic tool help function tools_print_help() { - tools_usage - echo - echo "Options:"; - echo " -f : collect forever : default: none" - echo " -p : overall collection period (minutes) : default: ${DEFAULT_PERIOD_MIN}" - if [ ${OPT_USE_INTERVALS} -eq 1 ]; then - echo " -i : sample interval (seconds) : default: ${DEFAULT_INTERVAL_SEC}" - fi - echo " -c : cpu list where tool runs (e.g., 0-1,8) : default: none" - echo - if [ ${OPT_USE_INTERVALS} -eq 1 ]; then - echo "Example: collect 5 minute period, sample every 30 seconds interval" - echo " ${TOOLNAME} -p 5 -i 30" - else - echo "Example: collect 5 minute period" - echo " ${TOOLNAME} -p 5" - fi + tools_usage + echo + echo "Options:"; + echo " -f : collect forever : default: none" + echo " -p : overall collection period (minutes) : default: ${DEFAULT_PERIOD_MIN}" + if [ ${OPT_USE_INTERVALS} -eq 1 ]; then + echo " -i : sample interval (seconds) : default: ${DEFAULT_INTERVAL_SEC}" + fi + echo " -c : cpu list where tool runs (e.g., 0-1,8) : default: none" + echo + if [ ${OPT_USE_INTERVALS} -eq 1 ]; then + echo "Example: collect 5 minute period, sample every 30 seconds interval" + echo " ${TOOLNAME} -p 5 -i 30" + else + echo "Example: collect 5 minute period" + echo " ${TOOLNAME} -p 5" + fi } # tools_parse_options() -- parse common options for tools scripts function tools_parse_options() { # check for no arguments, print usage - if [ $# -eq "0" ]; then - tools_usage - tools_cleanup 0 - exit 0 - fi + if [ $# -eq "0" ]; then + tools_usage + tools_cleanup 0 + exit 0 + fi # parse the input arguments - while getopts "fp:i:c:h" Option - do + while getopts "fp:i:c:h" Option + do case $Option in f) - OPT_FOREVER=1 - PERIOD_MIN=60 - ;; + OPT_FOREVER=1 + PERIOD_MIN=60 + ;; p) PERIOD_MIN=$OPTARG ;; i) - OPT_USE_INTERVALS=1 - INTERVAL_SEC=$OPTARG - ;; + OPT_USE_INTERVALS=1 + INTERVAL_SEC=$OPTARG + ;; c) CPULIST=$OPTARG ;; h) - tools_print_help - tools_cleanup 0 - exit 0 - ;; + tools_print_help + tools_cleanup 0 + exit 0 + ;; *) - tools_usage - tools_cleanup 0 - exit 0 - ;; + tools_usage + tools_cleanup 0 + exit 0 + ;; esac - done + done # validate input arguments - PERIOD_MAX=$[4*24*60] - INTERVAL_MAX=$[60*60] + PERIOD_MAX=$[4*24*60] + INTERVAL_MAX=$[60*60] - error=0 - if [[ ${PERIOD_MIN} -lt 1 || ${PERIOD_MIN} -gt ${PERIOD_MAX} ]]; then - echo "-p must be > 0 and <= ${PERIOD_MAX}." - error=1 - fi - if [[ ${INTERVAL_SEC} -lt 1 || ${INTERVAL_SEC} -gt ${INTERVAL_MAX} ]]; then - echo "-i must be > 0 and <= ${INTERVAL_MAX}." - error=1 - fi - if [ ${error} -eq 1 ]; then - tools_cleanup 0 - exit 1 - fi + error=0 + if [[ ${PERIOD_MIN} -lt 1 || ${PERIOD_MIN} -gt ${PERIOD_MAX} ]]; then + echo "-p must be > 0 and <= ${PERIOD_MAX}." + error=1 + fi + if [[ ${INTERVAL_SEC} -lt 1 || ${INTERVAL_SEC} -gt ${INTERVAL_MAX} ]]; then + echo "-i must be > 0 and <= ${INTERVAL_MAX}." + error=1 + fi + if [ ${error} -eq 1 ]; then + tools_cleanup 0 + exit 1 + fi } diff --git a/tools/engtools/hostdata-collectors/scripts/filestats.sh b/tools/engtools/hostdata-collectors/scripts/filestats.sh index 19d38a704..4f06b7562 100644 --- a/tools/engtools/hostdata-collectors/scripts/filestats.sh +++ b/tools/engtools/hostdata-collectors/scripts/filestats.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi PAGE_SIZE=$(getconf PAGE_SIZE) @@ -18,19 +18,19 @@ OPT_USE_INTERVALS=1 function print_files() { - print_separator + print_separator TOOL_HIRES_TIME - ${ECHO} "# ls -l /proc/*/fd" - sudo ls -l /proc/*/fd 2>/dev/null | awk \ - '$11 ~ /socket/ {a += 1} ; \ - $11 ~ /null/ {b += 1} ; \ - {c += 1} \ - END {\ - {printf "%-10s %-10s %-10s %-10s\n", "TOTAL", "FILES", "SOCKETS", "NULL PIPES"} \ - {printf "%-10s %-10s %-10s %-10s\n", c, c-(a+b) , a, b}}' + ${ECHO} "# ls -l /proc/*/fd" + sudo ls -l /proc/*/fd 2>/dev/null | awk \ + '$11 ~ /socket/ {a += 1} ; \ + $11 ~ /null/ {b += 1} ; \ + {c += 1} \ + END {\ + {printf "%-10s %-10s %-10s %-10s\n", "TOTAL", "FILES", "SOCKETS", "NULL PIPES"} \ + {printf "%-10s %-10s %-10s %-10s\n", c, c-(a+b) , a, b}}' - ${ECHO} + ${ECHO} ${ECHO} "# lsof" printf "%-7s %-7s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %s\n" "PID" "TOTAL" "FD" "U" "W" "R" "CWD" "RTD" "TXT" "MEM" "DEL" "TCP" "CMD" @@ -46,7 +46,7 @@ function print_files() {printf "%-7s %-7s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %s\n", \ pids[i]["PID"], \ pids[i]["TOTAL"],\ - ((pids[i]["u"]!="")? pids[i]["u"] : 0) + ((pids[i]["w"]!="")? pids[i]["w"] : 0 )+ ((pids[i]["r"]!="")? pids[i]["r"] : 0),\ + ((pids[i]["u"]!="")? pids[i]["u"] : 0) + ((pids[i]["w"]!="")? pids[i]["w"] : 0 )+ ((pids[i]["r"]!="")? pids[i]["r"] : 0),\ (pids[i]["u"]!="")? pids[i]["u"] : 0,\ (pids[i]["w"]!="")? pids[i]["w"] : 0,\ (pids[i]["r"]!="")? pids[i]["r"] : 0,\ @@ -56,13 +56,13 @@ function print_files() (pids[i]["mem"]!="")? pids[i]["mem"] : 0,\ (pids[i]["DEL"]!="")? pids[i]["DEL"] : 0,\ (pids[i]["TCP"]!="")? pids[i]["TCP"] : 0,\ - pids[i]["COMMAND"]} }}}' | sort -n -r -k3 + pids[i]["COMMAND"]} }}}' | sort -n -r -k3 - ${ECHO} + ${ECHO} - ${ECHO} "# lsof -nP +L1" - sudo lsof -nP +L1 - ${ECHO} + ${ECHO} "# lsof -nP +L1" + sudo lsof -nP +L1 + ${ECHO} } @@ -87,8 +87,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_files - sleep ${INTERVAL_SEC} + print_files + sleep ${INTERVAL_SEC} done print_files LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh b/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh index 6712e7976..f4c93ff67 100644 --- a/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh @@ -41,76 +41,76 @@ fi # Check for sufficient priviledges # [ JGAULD : possibly provide user = 'operator' option instead... ] if [ $UID -ne 0 ]; then - log_daemon_msg "Starting ${NAME} requires sudo/root access." - exit 1 + log_daemon_msg "Starting ${NAME} requires sudo/root access." + exit 1 fi case $1 in - start) + start) if [ -e ${PIDFILE} ]; then - pid=$(pidof -x ${NAME}) - if test "${pid}" != "" - then - echo_success "${NAME} already running" + pid=$(pidof -x ${NAME}) + if test "${pid}" != "" + then + echo_success "${NAME} already running" exit - fi + fi fi log_daemon_msg "Starting ${NAME}" if start-stop-daemon --start --background --quiet --oknodo --pidfile ${PIDFILE} \ - --exec ${DAEMON} -- ${DAEMON_ARGS} ; then - log_end_msg 0 + --exec ${DAEMON} -- ${DAEMON_ARGS} ; then + log_end_msg 0 else - log_end_msg 1 + log_end_msg 1 fi ;; - stop) + stop) if [ -e ${PIDFILE} ]; then - pids=$(pidof -x ${NAME}) - if [[ ! -z "${pids}" ]] - then - echo_success "Stopping ${NAME} [$pid]" - start-stop-daemon --stop --quiet --oknodo --pidfile ${PIDFILE} --retry=TERM/3/KILL/5 - # [ JGAULD: none of the following should be necessary ] - /usr/local/bin/cleanup-engtools.sh - else - echo_failure "${NAME} is not running" - fi + pids=$(pidof -x ${NAME}) + if [[ ! -z "${pids}" ]] + then + echo_success "Stopping ${NAME} [$pid]" + start-stop-daemon --stop --quiet --oknodo --pidfile ${PIDFILE} --retry=TERM/3/KILL/5 + # [ JGAULD: none of the following should be necessary ] + /usr/local/bin/cleanup-engtools.sh + else + echo_failure "${NAME} is not running" + fi else - echo_failure "${PIDFILE} does not exist" + echo_failure "${PIDFILE} does not exist" fi ;; - restart) + restart) $0 stop && sleep 2 && $0 start ;; - status) + status) if [ -e ${PIDFILE} ]; then - pid=$(pidof -x ${NAME}) - if test "${pid}" != "" - then - echo_success "${NAME} is running" - else - echo_success "${NAME} is not running" - fi + pid=$(pidof -x ${NAME}) + if test "${pid}" != "" + then + echo_success "${NAME} is running" + else + echo_success "${NAME} is not running" + fi else - echo_success "${NAME} is not running" + echo_success "${NAME} is not running" fi ;; - reload) + reload) if [ -e ${PIDFILE} ]; then - start-stop-daemon --stop --signal USR1 --quiet --pidfile ${PIDFILE} --name ${NAME} - echo_success "${NAME} reloaded successfully" + start-stop-daemon --stop --signal USR1 --quiet --pidfile ${PIDFILE} --name ${NAME} + echo_success "${NAME} reloaded successfully" else - echo_success "${PIDFILE} does not exist" + echo_success "${PIDFILE} does not exist" fi ;; - *) + *) echo "Usage: $0 {start|stop|restart|reload|status}" exit 2 ;; diff --git a/tools/engtools/hostdata-collectors/scripts/iostat.sh b/tools/engtools/hostdata-collectors/scripts/iostat.sh index 04be90c35..7ac65aceb 100644 --- a/tools/engtools/hostdata-collectors/scripts/iostat.sh +++ b/tools/engtools/hostdata-collectors/scripts/iostat.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -15,9 +15,9 @@ OPT_USE_INTERVALS=1 IOSTAT=$( which iostat 2>/dev/null ) if [ $? -ne 0 ]; then - print_separator - WARNLOG "iostat not available" - tools_cleanup 0 + print_separator + WARNLOG "iostat not available" + tools_cleanup 0 fi # MAIN Program: diff --git a/tools/engtools/hostdata-collectors/scripts/memstats.sh b/tools/engtools/hostdata-collectors/scripts/memstats.sh index 664b10b46..81e47777d 100644 --- a/tools/engtools/hostdata-collectors/scripts/memstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/memstats.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi PAGE_SIZE=$(getconf PAGE_SIZE) @@ -19,66 +19,66 @@ OPT_USE_INTERVALS=1 function print_memory() { # Configuration for netcmds - MEMINFO=/proc/meminfo - NODEINFO=/sys/devices/system/node/node?/meminfo - BUDDYINFO=/proc/buddyinfo - SLABINFO=/proc/slabinfo + MEMINFO=/proc/meminfo + NODEINFO=/sys/devices/system/node/node?/meminfo + BUDDYINFO=/proc/buddyinfo + SLABINFO=/proc/slabinfo - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME - ${ECHO} "# ${MEMINFO}" - ${CAT} ${MEMINFO} - ${ECHO} + ${ECHO} "# ${MEMINFO}" + ${CAT} ${MEMINFO} + ${ECHO} - ${ECHO} "# ${NODEINFO}" - ${CAT} ${NODEINFO} - ${ECHO} + ${ECHO} "# ${NODEINFO}" + ${CAT} ${NODEINFO} + ${ECHO} - ${ECHO} "# ${BUDDYINFO}" - ${CAT} ${BUDDYINFO} - ${ECHO} + ${ECHO} "# ${BUDDYINFO}" + ${CAT} ${BUDDYINFO} + ${ECHO} - ${ECHO} "# PSS" - cat /proc/*/smaps 2>/dev/null | \ + ${ECHO} "# PSS" + cat /proc/*/smaps 2>/dev/null | \ awk '/^Pss:/ {a += $2;} END {printf "%d MiB\n", a/1024.0;}' - ${ECHO} + ${ECHO} # use old slabinfo format (i.e. slub not enabled in kernel) - ${ECHO} "# ${SLABINFO}" - ${CAT} ${SLABINFO} | \ + ${ECHO} "# ${SLABINFO}" + ${CAT} ${SLABINFO} | \ awk -v page_size_B=${PAGE_SIZE} ' BEGIN {page_KiB = page_size_B/1024; TOT_KiB = 0;} (NF == 17) { - gsub(/[<>]/, ""); - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", + gsub(/[<>]/, ""); + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", $2, $3, $4, $5, $6, $7, $8, $10, $11, $12, $13, $15, $16, $17, "KiB"); } (NF == 16) { - num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; - KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; - TOT_KiB += KiB; - printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", + num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; + KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; + TOT_KiB += KiB; + printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", $1, $2, $3, $4, $5, $6, $7, $9, $10, $11, $12, $14, $15, $16, KiB); } END { - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", "TOTAL", "-", "-", "-", "-", "-", ":", "-", "-", "-", ":", "-", "-", "-", TOT_KiB); } ' 2>/dev/null - ${ECHO} + ${ECHO} - ${ECHO} "# disk usage: rootfs, tmpfs" - cmd='df -h -H -T --local -t rootfs -t tmpfs' - ${ECHO} "Disk space usage rootfs,tmpfs (SI):" - ${ECHO} "${cmd}" - ${cmd} - ${ECHO} + ${ECHO} "# disk usage: rootfs, tmpfs" + cmd='df -h -H -T --local -t rootfs -t tmpfs' + ${ECHO} "Disk space usage rootfs,tmpfs (SI):" + ${ECHO} "${cmd}" + ${cmd} + ${ECHO} - CMD='ps -e -o ppid,pid,nlwp,rss:10,vsz:10,cmd --sort=-rss' - ${ECHO} "# ${CMD}" - ${CMD} - ${ECHO} + CMD='ps -e -o ppid,pid,nlwp,rss:10,vsz:10,cmd --sort=-rss' + ${ECHO} "# ${CMD}" + ${CMD} + ${ECHO} } #------------------------------------------------------------------------------- @@ -101,8 +101,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_memory - sleep ${INTERVAL_SEC} + print_memory + sleep ${INTERVAL_SEC} done print_memory LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/netstats.sh b/tools/engtools/hostdata-collectors/scripts/netstats.sh index 4ed13b04e..cd306f743 100644 --- a/tools/engtools/hostdata-collectors/scripts/netstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/netstats.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -17,22 +17,22 @@ OPT_USE_INTERVALS=1 function print_netcmds() { # Configuration for netcmds - DEV=/proc/net/dev - NETSTAT=/proc/net/netstat + DEV=/proc/net/dev + NETSTAT=/proc/net/netstat - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME - for net in \ + for net in \ ${DEV} ${NETSTAT} - do - if [ -e "${net}" ] - then - ${ECHO} "# ${net}" - ${CAT} ${net} - ${ECHO} - fi - done + do + if [ -e "${net}" ] + then + ${ECHO} "# ${net}" + ${CAT} ${net} + ${ECHO} + fi + done } #------------------------------------------------------------------------------- @@ -55,8 +55,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_netcmds - sleep ${INTERVAL_SEC} + print_netcmds + sleep ${INTERVAL_SEC} done print_netcmds LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/postgres.sh b/tools/engtools/hostdata-collectors/scripts/postgres.sh index 9bcf8d1f3..7ee0a8577 100644 --- a/tools/engtools/hostdata-collectors/scripts/postgres.sh +++ b/tools/engtools/hostdata-collectors/scripts/postgres.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -16,32 +16,32 @@ OPT_USE_INTERVALS=1 # Print key networking device statistics function print_postgres() { - print_separator - TOOL_HIRES_TIME - + print_separator + TOOL_HIRES_TIME + # postgressql command: set user, disable pagination, and be quiet - PSQL="sudo -u postgres psql --pset pager=off -q" + PSQL="sudo -u postgres psql --pset pager=off -q" # List postgres databases - db_list=( $(${PSQL} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) - ${ECHO} "# postgres databases" - echo "db_list = ${db_list[@]}" - ${ECHO} + db_list=( $(${PSQL} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) + ${ECHO} "# postgres databases" + echo "db_list = ${db_list[@]}" + ${ECHO} # List sizes of all postgres databases (similar to "\l+") - ${ECHO} "# postgres database sizes" - ${PSQL} -c " + ${ECHO} "# postgres database sizes" + ${PSQL} -c " SELECT - pg_database.datname, - pg_database_size(pg_database.datname), - pg_size_pretty(pg_database_size(pg_database.datname)) + pg_database.datname, + pg_database_size(pg_database.datname), + pg_size_pretty(pg_database_size(pg_database.datname)) FROM pg_database ORDER BY pg_database_size DESC; " # For each database, list tables and their sizes (similar to "\dt+") - for db in "${db_list[@]}" - do + for db in "${db_list[@]}" + do ${ECHO} "# postgres database: ${db}" ${PSQL} -d ${db} -c " SELECT @@ -85,29 +85,29 @@ SELECT last_autoanalyze FROM pg_stat_user_tables; " - done + done # Specific table counts (This is very SLOW, look at "live tuples" instead) # Number of keystone tokens #${ECHO} "# keystone token count" # Number of postgres connections - ${ECHO} "# postgres database connections" - CONN=$(ps -C postgres -o cmd= | wc -l) - CONN_T=$(ps -C postgres -o cmd= | awk '/postgres: / {print $3}' | awk '{for(i=1;i<=NF;i++) a[$i]++} END {for(k in a) print k, a[k]}' | sort -k 2 -nr ) - ${ECHO} "connections total = ${CONN}" - ${ECHO} - ${ECHO} "connections breakdown:" - ${ECHO} "${CONN_T}" - ${ECHO} + ${ECHO} "# postgres database connections" + CONN=$(ps -C postgres -o cmd= | wc -l) + CONN_T=$(ps -C postgres -o cmd= | awk '/postgres: / {print $3}' | awk '{for(i=1;i<=NF;i++) a[$i]++} END {for(k in a) print k, a[k]}' | sort -k 2 -nr ) + ${ECHO} "connections total = ${CONN}" + ${ECHO} + ${ECHO} "connections breakdown:" + ${ECHO} "${CONN_T}" + ${ECHO} - ${ECHO} "connections breakdown (query):" - ${PSQL} -c "SELECT datname,state,count(*) from pg_stat_activity group by datname,state;" - ${ECHO} + ${ECHO} "connections breakdown (query):" + ${PSQL} -c "SELECT datname,state,count(*) from pg_stat_activity group by datname,state;" + ${ECHO} - ${ECHO} "connections idle age:" - ${PSQL} -c "SELECT datname,age(now(),state_change) from pg_stat_activity where state='idle';" - ${ECHO} + ${ECHO} "connections idle age:" + ${PSQL} -c "SELECT datname,age(now(),state_change) from pg_stat_activity where state='idle';" + ${ECHO} } #------------------------------------------------------------------------------- @@ -130,8 +130,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_postgres - sleep ${INTERVAL_SEC} + print_postgres + sleep ${INTERVAL_SEC} done print_postgres LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh b/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh index c588c16c5..64e0de9cb 100644 --- a/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh +++ b/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -17,8 +17,8 @@ MQOPT="-n rabbit@localhost" # Print key networking device statistics function print_rabbitmq() { - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME # IMPORTANT: # - Difficulty getting rabbitmqctl to work from init.d script; @@ -27,31 +27,31 @@ function print_rabbitmq() # - WORKAROUND: run command using 'sudo', even if you are 'root' # Dump various rabbitmq related stats - MQ_STATUS="rabbitmqctl ${MQOPT} status" - ${ECHO} "# ${MQ_STATUS}" - sudo ${MQ_STATUS} | grep -e '{memory' -A30 - ${ECHO} + MQ_STATUS="rabbitmqctl ${MQOPT} status" + ${ECHO} "# ${MQ_STATUS}" + sudo ${MQ_STATUS} | grep -e '{memory' -A30 + ${ECHO} # THe following is useful in diagnosing rabbit memory leaks # when end-users do not drain their queues (eg, due to RPC timeout issues, etc) - MQ_QUEUES="rabbitmqctl ${MQOPT} list_queues messages name pid messages_ready messages_unacknowledged memory consumers" - ${ECHO} "# ${MQ_QUEUES}" - sudo ${MQ_QUEUES} - ${ECHO} + MQ_QUEUES="rabbitmqctl ${MQOPT} list_queues messages name pid messages_ready messages_unacknowledged memory consumers" + ${ECHO} "# ${MQ_QUEUES}" + sudo ${MQ_QUEUES} + ${ECHO} - num_queues=$(sudo rabbitmqctl ${MQOPT} list_queues | wc -l); ((num_queues-=2)) - num_bindings=$(sudo rabbitmqctl ${MQOPT} list_bindings | wc -l); ((num_bindings-=2)) - num_exchanges=$(sudo rabbitmqctl ${MQOPT} list_exchanges | wc -l); ((num_exchanges-=2)) - num_connections=$(sudo rabbitmqctl ${MQOPT} list_connections | wc -l); ((num_connections-=2)) - num_channels=$(sudo rabbitmqctl ${MQOPT} list_channels | wc -l); ((num_channels-=2)) - arr=($(sudo rabbitmqctl ${MQOPT} list_queues messages consumers memory | \ - awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) - messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]} - printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" \ + num_queues=$(sudo rabbitmqctl ${MQOPT} list_queues | wc -l); ((num_queues-=2)) + num_bindings=$(sudo rabbitmqctl ${MQOPT} list_bindings | wc -l); ((num_bindings-=2)) + num_exchanges=$(sudo rabbitmqctl ${MQOPT} list_exchanges | wc -l); ((num_exchanges-=2)) + num_connections=$(sudo rabbitmqctl ${MQOPT} list_connections | wc -l); ((num_connections-=2)) + num_channels=$(sudo rabbitmqctl ${MQOPT} list_channels | wc -l); ((num_channels-=2)) + arr=($(sudo rabbitmqctl ${MQOPT} list_queues messages consumers memory | \ + awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) + messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]} + printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" \ "queues" "bindings" "exchanges" "connections" "channels" "messages" "consumers" "memory" - printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" \ + printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" \ $num_queues $num_bindings $num_exchanges $num_connections $num_channels $messages $consumers $memory - ${ECHO} + ${ECHO} } #------------------------------------------------------------------------------- @@ -74,8 +74,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_rabbitmq - sleep ${INTERVAL_SEC} + print_rabbitmq + sleep ${INTERVAL_SEC} done print_rabbitmq LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh b/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh index 3d972c477..e5ccd1ab2 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -22,24 +22,24 @@ STORAGE=( $(system host-list | awk '(/storage/) {print $4;}') ) LOG "Remote bzip2 engtools data on all blades:" for blade in ${CONTROLLER[@]}; do - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "bzip2 on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo bzip2 /scratch/syseng_data/${blade}/* - else - WARNLOG "cannot ping: ${blade}" - fi + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "bzip2 on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo bzip2 /scratch/syseng_data/${blade}/* + else + WARNLOG "cannot ping: ${blade}" + fi done for blade in ${STORAGE[@]} ${COMPUTE[@]} ; do - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "bzip2 on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo bzip2 /tmp/syseng_data/${blade}/* - else - WARNLOG "cannot ping: ${blade}" - fi + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "bzip2 on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo bzip2 /tmp/syseng_data/${blade}/* + else + WARNLOG "cannot ping: ${blade}" + fi done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh b/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh index f3df76d55..21a47a4f3 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -18,19 +18,19 @@ BLADES=( $(system host-list | awk '(/compute|controller|storage/) {print $4;}') LOG "Remote start engtools on all blades:" for blade in ${BLADES[@]}; do - if [ "${blade}" == "${HOSTNAME}" ]; then - LOG "start on $blade:" - sudo service collect-engtools.sh start - else - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "start on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo service collect-engtools.sh start + if [ "${blade}" == "${HOSTNAME}" ]; then + LOG "start on $blade:" + sudo service collect-engtools.sh start else - WARNLOG "cannot ping: ${blade}" + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "start on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo service collect-engtools.sh start + else + WARNLOG "cannot ping: ${blade}" + fi fi - fi done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh b/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh index 1251ea827..5637bfd22 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -18,19 +18,19 @@ BLADES=( $(system host-list | awk '(/compute|controller|storage/) {print $4;}') LOG "Remote stop engtools on all blades:" for blade in ${BLADES[@]}; do - if [ "${blade}" == "${HOSTNAME}" ]; then - LOG "stop on $blade:" - sudo service collect-engtools.sh stop - else - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "stop on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo service collect-engtools.sh stop + if [ "${blade}" == "${HOSTNAME}" ]; then + LOG "stop on $blade:" + sudo service collect-engtools.sh stop else - WARNLOG "cannot ping: ${blade}" + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "stop on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo service collect-engtools.sh stop + else + WARNLOG "cannot ping: ${blade}" + fi fi - fi done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh b/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh index 6f82f47e7..2e6704293 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -24,10 +24,10 @@ STORAGE=( $(system host-list | awk '(/storage/) {print $4;}') ) DEST=/opt/backups/syseng_data/ if [[ "${HOSTNAME}" =~ "controller-" ]]; then - LOG "rsync DEST=${DEST}" + LOG "rsync DEST=${DEST}" else - LOG "*ERROR* only run this on controller" - exit 1 + LOG "*ERROR* only run this on controller" + exit 1 fi sudo mkdir -p ${DEST} @@ -43,13 +43,13 @@ SRC=/scratch/syseng_data/ DEST=/opt/backups/syseng_data/ for HOST in ${CONTROLLER[@]} do - ping -c1 ${HOST} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" - sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} - else - WARNLOG "cannot ping: ${HOST}" - fi + ping -c1 ${HOST} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" + sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} + else + WARNLOG "cannot ping: ${HOST}" + fi done # computes & storage @@ -57,13 +57,13 @@ SRC=/tmp/syseng_data/ DEST=/opt/backups/syseng_data/ for HOST in ${STORAGE[@]} ${COMPUTE[@]} do - ping -c1 ${HOST} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" - sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} - else - WARNLOG "cannot ping: ${HOST}" - fi + ping -c1 ${HOST} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" + sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} + else + WARNLOG "cannot ping: ${HOST}" + fi done LOG 'done' diff --git a/tools/engtools/hostdata-collectors/scripts/slab.sh b/tools/engtools/hostdata-collectors/scripts/slab.sh index 70e9c0596..f60e0c095 100644 --- a/tools/engtools/hostdata-collectors/scripts/slab.sh +++ b/tools/engtools/hostdata-collectors/scripts/slab.sh @@ -3,19 +3,19 @@ PAGE_SIZE=$(getconf PAGE_SIZE) cat /proc/slabinfo | awk -v page_size_B=${PAGE_SIZE} ' BEGIN {page_KiB = page_size_B/1024; TOT_KiB = 0;} (NF == 17) { - gsub(/[<>]/, ""); - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", + gsub(/[<>]/, ""); + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", $2, $3, $4, $5, $6, $7, $8, $10, $11, $12, $13, $15, $16, $17, "KiB"); } (NF == 16) { - num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; - KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; - TOT_KiB += KiB; - printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", + num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; + KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; + TOT_KiB += KiB; + printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", $1, $2, $3, $4, $5, $6, $7, $9, $10, $11, $12, $14, $15, $16, KiB); } END { - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", "TOTAL", "-", "-", "-", "-", "-", ":", "-", "-", "-", ":", "-", "-", "-", TOT_KiB); } ' 2>/dev/null diff --git a/tools/engtools/hostdata-collectors/scripts/ticker.sh b/tools/engtools/hostdata-collectors/scripts/ticker.sh index 570cd526d..0bfe8ad68 100644 --- a/tools/engtools/hostdata-collectors/scripts/ticker.sh +++ b/tools/engtools/hostdata-collectors/scripts/ticker.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -31,15 +31,15 @@ LOG_NOCR "collecting " t=0 for ((rep=1; rep <= REPEATS ; rep++)) do - ((t++)) - sleep ${INTERVAL_SEC} - if [ ${t} -ge ${REP_LOG} ]; then - t=0 - echo "." - LOG_NOCR "collecting " - else - echo -n "." - fi + ((t++)) + sleep ${INTERVAL_SEC} + if [ ${t} -ge ${REP_LOG} ]; then + t=0 + echo "." + LOG_NOCR "collecting " + else + echo -n "." + fi done echo "." diff --git a/tools/engtools/hostdata-collectors/scripts/top.sh b/tools/engtools/hostdata-collectors/scripts/top.sh index 45dff33d7..9a5ebc0cd 100644 --- a/tools/engtools/hostdata-collectors/scripts/top.sh +++ b/tools/engtools/hostdata-collectors/scripts/top.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval diff --git a/tools/engtools/hostdata-collectors/scripts/vswitch.sh b/tools/engtools/hostdata-collectors/scripts/vswitch.sh index dae8bf76a..1e6a5a338 100644 --- a/tools/engtools/hostdata-collectors/scripts/vswitch.sh +++ b/tools/engtools/hostdata-collectors/scripts/vswitch.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -16,25 +16,25 @@ OPT_USE_INTERVALS=1 # Print key networking device statistics function print_vswitch() { - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME - cmd='vshell engine-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell engine-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell port-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell port-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell network-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell network-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell interface-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell interface-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell engine-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell engine-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell port-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell port-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell network-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell network-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell interface-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell interface-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} } #------------------------------------------------------------------------------- @@ -57,8 +57,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_vswitch - sleep ${INTERVAL_SEC} + print_vswitch + sleep ${INTERVAL_SEC} done print_vswitch LOG "done" diff --git a/tools/engtools/parsers/common/cleanup-uncompressed.sh b/tools/engtools/parsers/common/cleanup-uncompressed.sh index 3bd1fa21f..26824b269 100755 --- a/tools/engtools/parsers/common/cleanup-uncompressed.sh +++ b/tools/engtools/parsers/common/cleanup-uncompressed.sh @@ -5,15 +5,15 @@ #SPDX-License-Identifier: Apache-2.0 # # This script removes uncompressed file. It can save a huge amount of disk space -# on the analysis server. Run this script after the very last time the data is parsed -# and BEFORE running parse-daily.sh script. -# If it is run after each intermediary parse, the download-data.sh script will download the +# on the analysis server. Run this script after the very last time the data is parsed +# and BEFORE running parse-daily.sh script. +# If it is run after each intermediary parse, the download-data.sh script will download the # uncompressed files again. if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf diff --git a/tools/engtools/parsers/common/download-data.sh b/tools/engtools/parsers/common/download-data.sh index 0c5726dd1..7ac8c3d21 100755 --- a/tools/engtools/parsers/common/download-data.sh +++ b/tools/engtools/parsers/common/download-data.sh @@ -14,9 +14,9 @@ # This script is to be run after running download-computes.sh on one of the controllers. if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf @@ -38,6 +38,6 @@ for HOST in ${ALL_HOSTS}; do bzip2 ${HOST}* cd ${CURDIR} else - echo "${HOST} not found" + echo "${HOST} not found" fi done diff --git a/tools/engtools/parsers/common/parse-all.sh b/tools/engtools/parsers/common/parse-all.sh index 99d1e4eb5..42f732dcc 100755 --- a/tools/engtools/parsers/common/parse-all.sh +++ b/tools/engtools/parsers/common/parse-all.sh @@ -8,7 +8,7 @@ # This script is used to parse all stats data. It is designed to be called by either # parse-controllers.sh or parse-computes.sh and not used as a standalone script. # If the input node is a controller, it will parse controller specific postgres & -# and rabbitmq stats first. If the input node is a compute, it will pars the compute +# and rabbitmq stats first. If the input node is a compute, it will pars the compute # specific vswitch stats first. # # The following parsing steps are common to all hosts and are executed in the specified order: @@ -22,12 +22,12 @@ # - Parse filestats (summary) # - Parse process level schedtop (optional step, configured in lab.conf) # - Generate tarball - + if [[ $# != 1 ]]; then - echo "ERROR: This script is meant to be called by either parse-controllers.sh or parse-computes.sh script." - echo "To run it separately, copy the script to the host directory that contains *.bz2 files." - echo "It takes a single argument - the name of the host directory (e.g. ./parse-all.sh controller-0)." - exit 1 + echo "ERROR: This script is meant to be called by either parse-controllers.sh or parse-computes.sh script." + echo "To run it separately, copy the script to the host directory that contains *.bz2 files." + echo "It takes a single argument - the name of the host directory (e.g. ./parse-all.sh controller-0)." + exit 1 fi source ../lab.conf @@ -49,7 +49,7 @@ function sedit() # Remove any trailing comma sed -i "s/,$//" ${FILETOSED} } - + function get_filename_from_mountname() { local name=$1 @@ -153,7 +153,7 @@ function parse_occtop_data() echo "${header}" > occtop-${NODE}-detailed.csv cat tmp2.txt >> occtop-${NODE}-detailed.csv - # Generate simple CSV file which is used to generate host CPU occupancy chart. Platform cores are + # Generate simple CSV file which is used to generate host CPU occupancy chart. Platform cores are # defined in the host.conf. The simple CSV contains only the Date/Time and Total platform CPU occupancy e.g. # Date/Time,Total # 2016-11-22 00:29:16.523,94.9 @@ -187,7 +187,7 @@ function parse_memtop_data() bzcat *memtop.bz2 > memtop-${NODE}-${DATESTAMP}.txt cp memtop-${NODE}-${DATESTAMP}.txt tmp.txt sedit tmp.txt - + # After dumping all memtop bz2 output into one text file and in-place sed, grab only relevant data # for CSV output. Generate both detailed and simple CSV files. Simple output will be used to generate # chart. @@ -222,7 +222,7 @@ function parse_netstats_data() done < tmp.txt done rm tmp.txt - fi + fi } function parse_iostats_data() @@ -238,28 +238,28 @@ function parse_iostats_data() bzcat *iostat.bz2 | grep -E "/2015|/2016|/2017|${DEVICE}" | awk '{print $1","$2","$3","$4","$5","$6","$7","$8","$9","$10","$11","$12","$13","$14}' > tmp.txt while IFS= read -r current do - if test "${current#*Linux}" != "$current" - then - # Skip the line that contains the word "Linux" - continue - else - if test "${current#*$DEVICE}" == "$current" - then - # It's a date entry, look ahead - read -r next - if test "${next#*$DEVICE}" != "${next}" - then - # This next line contains the device stats - # Combine date and time fields - current="${current//2016,/2016 }" - current="${current//2017,/2017 }" - # Combine time and AM/PM fields - current="${current//,AM/ AM}" - current="${current//,PM/ PM}" - # Write both lines to intermediate file - echo "${current}" >> tmp2.txt - echo "${next}" >> tmp2.txt - fi + if test "${current#*Linux}" != "$current" + then + # Skip the line that contains the word "Linux" + continue + else + if test "${current#*$DEVICE}" == "$current" + then + # It's a date entry, look ahead + read -r next + if test "${next#*$DEVICE}" != "${next}" + then + # This next line contains the device stats + # Combine date and time fields + current="${current//2016,/2016 }" + current="${current//2017,/2017 }" + # Combine time and AM/PM fields + current="${current//,AM/ AM}" + current="${current//,PM/ PM}" + # Write both lines to intermediate file + echo "${current}" >> tmp2.txt + echo "${next}" >> tmp2.txt + fi fi fi done < tmp.txt @@ -272,7 +272,7 @@ function parse_iostats_data() cut -d, -f2-11 --complement tmp2.txt > tmp.txt # Write final content to output csv cat tmp.txt >> iostat-${NODE}-${DEVICE}.csv - rm tmp.txt tmp2.txt + rm tmp.txt tmp2.txt done fi } @@ -317,9 +317,9 @@ parse_occtop_data # Parsing memtop data parse_memtop_data -# Parsing memstats data to generate the high level report. The most important piece of info is the list of +# Parsing memstats data to generate the high level report. The most important piece of info is the list of # hi-runners at the end of the file. If there is a leak, run parse-daily.sh script to generate the time -# series data for the offending processes only. Use process name, not PID as most Titanium Cloud processes have +# series data for the offending processes only. Use process name, not PID as most Titanium Cloud processes have # workers. LOG "Parsing memstats summary for ${NODE}" ../parse_memstats --report *memstats.bz2 > memstats-summary-${NODE}-${DATESTAMP}.txt @@ -331,7 +331,7 @@ rm pid-*.csv parse_netstats_data # Parsing schedtop data to generate the high level report. Leave the process level schedtop parsing till -# the end as it is a long running task. +# the end as it is a long running task. LOG "Parsing schedtop summary for ${NODE}" FILES=$(ls *schedtop.bz2) ../parse_schedtop ${FILES} > schedtop-summary-${NODE}-${DATESTAMP}.txt @@ -342,17 +342,17 @@ parse_iostats_data # Parsing diskstats data parse_diskstats_data -# Parsing filestats data to generate the high level report. If there is a file descriptor leak, run parse-daily.sh -# script to generate the time series data for the offending processes only. Use process name, not PID as most +# Parsing filestats data to generate the high level report. If there is a file descriptor leak, run parse-daily.sh +# script to generate the time series data for the offending processes only. Use process name, not PID as most # Titanium Cloud processes have workers. LOG "Parsing filestats summary for ${NODE}" -../parse_filestats --all *filestats.bz2 > filestats-summary-${NODE}-${DATESTAMP}.txt +../parse_filestats --all *filestats.bz2 > filestats-summary-${NODE}-${DATESTAMP}.txt # Parsing process level schedtop data. This is a long running task. To skip this step or generate data for # only specific processes, update the lab.conf and host.conf files. [[ ${GENERATE_PROCESS_SCHEDTOP} == Y ]] && parse_process_schedtop_data || WARNLOG "Parsing process level schedtop is skipped." -# Done parsing for this host. If it's a controller host, check if the parsing of postgres connection stats which is run in +# Done parsing for this host. If it's a controller host, check if the parsing of postgres connection stats which is run in # parallel is done before creating a tar file. if test "${NODE#*"controller"}" != "${NODE}"; then # If postgres-conns.csv file has not been created which is highly unlikely, wait a couple of minutes diff --git a/tools/engtools/parsers/common/parse-controllers.sh b/tools/engtools/parsers/common/parse-controllers.sh index 61a77c585..da52b07ea 100755 --- a/tools/engtools/parsers/common/parse-controllers.sh +++ b/tools/engtools/parsers/common/parse-controllers.sh @@ -14,9 +14,9 @@ PARSERDIR=$(dirname $0) . ${PARSERDIR}/parse-util.sh if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf @@ -36,12 +36,12 @@ for HOST in ${CONTROLLER_LIST}; do sleep 120 cd .. else - ERRLOG "${HOST} does not exist. Parsing skipped." + ERRLOG "${HOST} does not exist. Parsing skipped." fi done # Parsing postgres connection stats is a time consuming step, run it in parallel with parse-all -# script. +# script. for HOST in ${CONTROLLER_LIST}; do if [ -d ${HOST} ]; then LOG "Parsing postgres connection stats data for controller host ${HOST}" diff --git a/tools/engtools/parsers/common/parse-daily.sh b/tools/engtools/parsers/common/parse-daily.sh index 7298441c3..971c5acdd 100755 --- a/tools/engtools/parsers/common/parse-daily.sh +++ b/tools/engtools/parsers/common/parse-daily.sh @@ -102,8 +102,8 @@ elif [[ $# -eq 3 ]]; then cd $1 parse_daily_stats $2 $3 else - echo "ERROR: Specified host $1 does not exist." - exit 1 + echo "ERROR: Specified host $1 does not exist." + exit 1 fi else echo "Specified parser $2 is not a valid parser." diff --git a/tools/engtools/parsers/common/parse-postgres.sh b/tools/engtools/parsers/common/parse-postgres.sh index 2e15e12ca..0d1a68d36 100755 --- a/tools/engtools/parsers/common/parse-postgres.sh +++ b/tools/engtools/parsers/common/parse-postgres.sh @@ -63,7 +63,7 @@ function generate_grep_str() { local grepstr="time:" for DB in ${DATABASE_LIST}; do - grepstr="${grepstr}|${DB}" + grepstr="${grepstr}|${DB}" done grepstr="${grepstr}|breakdown|connections total|rows" echo $grepstr diff --git a/tools/engtools/parsers/common/parse-rabbitmq.sh b/tools/engtools/parsers/common/parse-rabbitmq.sh index a5f19473d..01df49a8e 100755 --- a/tools/engtools/parsers/common/parse-rabbitmq.sh +++ b/tools/engtools/parsers/common/parse-rabbitmq.sh @@ -8,7 +8,7 @@ OUTFILE=$1 FILES=$(ls *rabbitmq.bz2 | sort) -[ -e tmp.txt ] && rm tmp.txt +[ -e tmp.txt ] && rm tmp.txt echo "Time/Date,Memory Total,Connection Readers,Connection Writers,Connection Channels,Connection Other,Queue Procs,Queue Slave Procs,Plugins,Other Proc,Mnesia,Mgmt DB,Msg Index,Other ETS,Binary,Code,Atom,Other System,VM Memory High Watermark,VM Memory Limit,Disk Free Limit,Disk Free,Descriptor Limit,Descriptors Used,Sockets Limit,Sockets Used,Processes Limit,Processes Used,Run Queue,Uptime" >${OUTFILE} diff --git a/tools/engtools/parsers/common/parse-schedtop.sh b/tools/engtools/parsers/common/parse-schedtop.sh index 90cd88346..cba3fcbba 100755 --- a/tools/engtools/parsers/common/parse-schedtop.sh +++ b/tools/engtools/parsers/common/parse-schedtop.sh @@ -48,7 +48,7 @@ function parse_schedtop_data() if [[ $# -eq 0 ]]; then # Parsing detail schedtop stats for all services configured in host.conf for all hosts would - # take a very long time and is often unnecessary. Until the performance issue with parse_schedtop + # take a very long time and is often unnecessary. Until the performance issue with parse_schedtop # is addressed, this is not supported. print_usage else diff --git a/tools/engtools/parsers/common/parse-util.sh b/tools/engtools/parsers/common/parse-util.sh index fa0c8f6d4..c469d1502 100755 --- a/tools/engtools/parsers/common/parse-util.sh +++ b/tools/engtools/parsers/common/parse-util.sh @@ -10,17 +10,17 @@ LOGFILE="parserlog.txt" function LOG () { - local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) - echo -e "${tstamp_H} $0($$): $@" >> ${LOGFILE} + local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) + echo -e "${tstamp_H} $0($$): $@" >> ${LOGFILE} } function ERRLOG () { - LOG "ERROR: $@" + LOG "ERROR: $@" } function WARNLOG () { - LOG "WARN: $@" + LOG "WARN: $@" } diff --git a/tools/engtools/parsers/large_office/download-computes.sh b/tools/engtools/parsers/large_office/download-computes.sh index 9e4806c01..fbceb79e3 100755 --- a/tools/engtools/parsers/large_office/download-computes.sh +++ b/tools/engtools/parsers/large_office/download-computes.sh @@ -6,14 +6,14 @@ # # Create /opt/backups/tmp/syseng-data directory on controller 0, change mode of this # directory to 777 and place this script and the lab.conf files there. It is recommended -# to set up password-less login from the controller to all storage and compute hosts +# to set up password-less login from the controller to all storage and compute hosts # before running the script. # if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf diff --git a/tools/engtools/parsers/large_office/parse-computes.sh b/tools/engtools/parsers/large_office/parse-computes.sh index 6addeaff4..60206c870 100755 --- a/tools/engtools/parsers/large_office/parse-computes.sh +++ b/tools/engtools/parsers/large_office/parse-computes.sh @@ -7,22 +7,22 @@ # This script is used to parse stats data for storage and compute hosts. It is not # relevant for CPE. For large office, it is called by parse-everything.sh script. -# File lab.conf must exist for the script to run. The STORAGE_LIST and COMPUTE_LIST -# config parameters can be set to suit the parsing needs. +# File lab.conf must exist for the script to run. The STORAGE_LIST and COMPUTE_LIST +# config parameters can be set to suit the parsing needs. PARSERDIR=$(dirname $0) . ${PARSERDIR}/parse-util.sh if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf if [ -z "${STORAGE_LIST}" ]; then - # This script could be invoked from another script or run separately so write to both + # This script could be invoked from another script or run separately so write to both # console and log file. echo "STORAGE_LIST is not set in lab.conf file. Skipping stats parsing for all storage." WARNLOG "STORAGE_LIST is not set in lab.conf file. Skipping stats parsing for all storage." @@ -35,7 +35,7 @@ else ../parse-all.sh ${HOST} > /dev/null 2>&1 & cd .. else - ERRLOG "${HOST} does not exist. Parsing skipped." + ERRLOG "${HOST} does not exist. Parsing skipped." fi done fi @@ -45,7 +45,7 @@ if [ -z "${COMPUTE_LIST}" ]; then WARNLOG "COMPUTE_LIST is not set in lab.conf file. Skipping stats parsing for all computes." exit 1 else - # If there is a large number of computes, they need to be parsed one batch at a time, + # If there is a large number of computes, they need to be parsed one batch at a time, # otherwise, the analysis server will be brought down to a crawl. Set the number of # computes to process in parallel as batches of 25 if it's not set in lab.conf BATCH_SIZE=${BATCH_SIZE:-25} diff --git a/utilities/build-info/build-info-1.0/collect.sh b/utilities/build-info/build-info-1.0/collect.sh index 8d06f6a62..48abc23d0 100755 --- a/utilities/build-info/build-info-1.0/collect.sh +++ b/utilities/build-info/build-info-1.0/collect.sh @@ -24,7 +24,7 @@ if [ ! -e $jenkinsBuildFile ]; then fi if [ -e $releaseInfoFile ]; then - source $releaseInfoFile + source $releaseInfoFile fi if [ "${REPO}x" == "x" ]; then @@ -35,12 +35,12 @@ if [ -e $jenkinsBuildFile ]; then cp $jenkinsBuildFile $destFile source $jenkinsBuildFile else - # PLATFORM_RELEASE should be set in release-info.inc - if [ "x${PLATFORM_RELEASE}" == "x" ]; then - SW_VERSION="Unknown" - else - SW_VERSION="${PLATFORM_RELEASE}" - fi + # PLATFORM_RELEASE should be set in release-info.inc + if [ "x${PLATFORM_RELEASE}" == "x" ]; then + SW_VERSION="Unknown" + else + SW_VERSION="${PLATFORM_RELEASE}" + fi BUILD_TARGET="Unknown" BUILD_TYPE="Informal" @@ -89,8 +89,8 @@ fi echo "#ifndef _BUILD_INFO_H_" > $destH echo "#define _BUILD_INFO_H_" >> $destH echo "" >> $destH -echo "#define RELEASE_NAME \"$RELEASE_NAME\"" >> $destH -echo "#define SW_VERSION \"$SW_VERSION\"" >> $destH +echo "#define RELEASE_NAME \"$RELEASE_NAME\"" >> $destH +echo "#define SW_VERSION \"$SW_VERSION\"" >> $destH echo "" >> $destH echo "#define BUILD_TARGET \"$BUILD_TARGET\"" >> $destH echo "#define BUILD_TYPE \"$BUILD_TYPE\"" >> $destH diff --git a/utilities/platform-util/scripts/cgcs_tc_setup.sh b/utilities/platform-util/scripts/cgcs_tc_setup.sh index 0630fac68..9507ae978 100755 --- a/utilities/platform-util/scripts/cgcs_tc_setup.sh +++ b/utilities/platform-util/scripts/cgcs_tc_setup.sh @@ -32,7 +32,7 @@ fi function test_valid_speed { # After the link is enabled but before the autonegotiation is complete - # the link speed may be read as either -1 or as 4294967295 (which is + # the link speed may be read as either -1 or as 4294967295 (which is # uint(-1) in twos-complement) depending on the kernel. Neither one is valid. if (( $1 > 0 )) && (( $1 != 4294967295 )) then @@ -62,16 +62,16 @@ function is_consolidated { if ! infra_exists then - return 1 + return 1 else # determine whether the management interface is a parent of the # infrastructure interface based on name. # eg. this matches enp0s8 to enp0s8.10 but not enp0s88 - if [[ $infrastructure_interface =~ $management_interface[\.][0-9]+$ ]] - then - return 0 - fi - return 1 + if [[ $infrastructure_interface =~ $management_interface[\.][0-9]+$ ]] + then + return 0 + fi + return 1 fi } diff --git a/utilities/platform-util/scripts/remotelogging_tc_setup.sh b/utilities/platform-util/scripts/remotelogging_tc_setup.sh index 3f8f8e459..4f44fa240 100755 --- a/utilities/platform-util/scripts/remotelogging_tc_setup.sh +++ b/utilities/platform-util/scripts/remotelogging_tc_setup.sh @@ -34,7 +34,7 @@ function log function test_valid_speed { # After the link is enabled but before the autonegotiation is complete - # the link speed may be read as either -1 or as 4294967295 (which is + # the link speed may be read as either -1 or as 4294967295 (which is # uint(-1) in twos-complement) depending on the kernel. Neither one is valid. if (( $1 > 0 )) && (( $1 != 4294967295 )) then diff --git a/virt/qemu/centos/files/build_configure.sh b/virt/qemu/centos/files/build_configure.sh index 31d78a19c..f9ba586f4 100755 --- a/virt/qemu/centos/files/build_configure.sh +++ b/virt/qemu/centos/files/build_configure.sh @@ -41,11 +41,11 @@ shift if [ "$have_rbd" == "enable" ]; then - rbd_driver=rbd, + rbd_driver=rbd, fi if [ "$have_gluster" == "enable" ]; then - gluster_driver=gluster, + gluster_driver=gluster, fi ./configure \