From 847cf1e92f9a966d6b03792fde7f3c3a5a40fce2 Mon Sep 17 00:00:00 2001 From: Sun Austin Date: Thu, 30 Aug 2018 14:50:33 +0800 Subject: [PATCH 1/9] Fix linters issue E002 and E003 and linters check (none-voting) E002 Tab indents E003 Indent not multiple of 4 Story: 2003366 Task: 24424 Change-Id: Ie063ccfe346489f08c4752adaa344f5919db0378 Signed-off-by: Sun Austin --- .zuul.yaml | 2 + base/haproxy/haproxy/haproxy.sh | 18 +- base/initscripts/centos/files/mountnfs.sh | 94 +-- .../centos/files/mysql-check-upgrade.sh | 6 +- .../centos/files/mysql-prepare-db-dir.sh | 4 +- .../centos/files/mysql-scripts-common.sh | 24 +- .../mariadb/centos/files/mysql-wait-ready.sh | 42 +- .../mariadb/centos/files/mysql-wait-stop.sh | 22 +- filesystem/nfscheck/files/nfscheck-init.sh | 4 +- .../lldpd/centos/files/i40e-lldp-configure.sh | 24 +- .../mlx4-config/files/mlx4-configure.sh | 12 +- .../mlx4-config/files/mlx4_core_config.sh | 2 +- tools/collector/scripts/collect_nfv_vim.sh | 2 +- tools/collector/scripts/collect_openstack.sh | 2 +- tools/collector/scripts/collect_psqldb.sh | 4 +- .../hostdata-collectors/patch-engtools.sh | 10 +- .../hostdata-collectors/scripts/ceph.sh | 24 +- .../scripts/cleanup-engtools.sh | 20 +- .../scripts/collect-engtools.sh | 234 ++++---- .../hostdata-collectors/scripts/diskstats.sh | 108 ++-- .../scripts/engtools_util.sh | 554 +++++++++--------- .../hostdata-collectors/scripts/filestats.sh | 40 +- .../scripts/init.d/collect-engtools.sh | 76 +-- .../hostdata-collectors/scripts/iostat.sh | 10 +- .../hostdata-collectors/scripts/memstats.sh | 84 +-- .../hostdata-collectors/scripts/netstats.sh | 34 +- .../hostdata-collectors/scripts/postgres.sh | 68 +-- .../hostdata-collectors/scripts/rabbitmq.sh | 50 +- .../scripts/remote/rbzip2-engtools.sh | 36 +- .../scripts/remote/rstart-engtools.sh | 26 +- .../scripts/remote/rstop-engtools.sh | 26 +- .../scripts/remote/rsync-engtools-data.sh | 38 +- .../hostdata-collectors/scripts/slab.sh | 14 +- .../hostdata-collectors/scripts/ticker.sh | 22 +- .../hostdata-collectors/scripts/top.sh | 4 +- .../hostdata-collectors/scripts/vswitch.sh | 44 +- .../parsers/common/cleanup-uncompressed.sh | 12 +- .../engtools/parsers/common/download-data.sh | 8 +- tools/engtools/parsers/common/parse-all.sh | 80 +-- .../parsers/common/parse-controllers.sh | 10 +- tools/engtools/parsers/common/parse-daily.sh | 4 +- .../engtools/parsers/common/parse-postgres.sh | 2 +- .../engtools/parsers/common/parse-rabbitmq.sh | 2 +- .../engtools/parsers/common/parse-schedtop.sh | 2 +- tools/engtools/parsers/common/parse-util.sh | 8 +- .../parsers/large_office/download-computes.sh | 8 +- .../parsers/large_office/parse-computes.sh | 16 +- .../build-info/build-info-1.0/collect.sh | 18 +- .../platform-util/scripts/cgcs_tc_setup.sh | 14 +- .../scripts/remotelogging_tc_setup.sh | 2 +- virt/qemu/centos/files/build_configure.sh | 4 +- 51 files changed, 988 insertions(+), 986 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 03a8bd5c8..549078afd 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -4,3 +4,5 @@ jobs: - openstack-tox-pep8: voting: false + - openstack-tox-linters: + voting: false diff --git a/base/haproxy/haproxy/haproxy.sh b/base/haproxy/haproxy/haproxy.sh index 560480e40..dc7e6ea92 100755 --- a/base/haproxy/haproxy/haproxy.sh +++ b/base/haproxy/haproxy/haproxy.sh @@ -7,7 +7,7 @@ # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: HA-Proxy TCP/HTTP reverse proxy -# Description: HA-Proxy is a TCP/HTTP reverse proxy +# Description: HA-Proxy is a TCP/HTTP reverse proxy ### END INIT INFO PATH=/sbin:/bin:/usr/sbin:/usr/bin @@ -22,12 +22,12 @@ RETVAL=0 # This is only needed till TPM In-Kernel # ResourceMgr comes in remove_TPM_transients () { - _HANDLES=`find $TPM_DATA_DIR -type f -name "hp*.bin" -printf "%f "` - for handle in $_HANDLES; do + _HANDLES=`find $TPM_DATA_DIR -type f -name "hp*.bin" -printf "%f "` + for handle in $_HANDLES; do handle_addr=`echo $handle | sed 's/hp\([0-9]*\)\.bin/\1/g'` tss2_flushcontext -ha $handle_addr &> /dev/null - done - rm -f $TPM_DATA_DIR/* + done + rm -f $TPM_DATA_DIR/* } start() { @@ -60,9 +60,9 @@ start() { stop() { if [ ! -e $PIDFILE ]; then return; fi - + echo -n "Stopping $DESC..." - + start-stop-daemon --stop --quiet --retry 3 --oknodo --pidfile $PIDFILE -x "$DAEMON" if [ -n "`pidof $DAEMON`" ] ; then pkill -KILL -f $DAEMON @@ -90,7 +90,7 @@ status() } check() { - /usr/sbin/$NAME -c -q -V -f /etc/$NAME/$NAME.cfg + /usr/sbin/$NAME -c -q -V -f /etc/$NAME/$NAME.cfg } # See how we were called. @@ -116,5 +116,5 @@ case "$1" in RETVAL=1 ;; esac - + exit $RETVAL diff --git a/base/initscripts/centos/files/mountnfs.sh b/base/initscripts/centos/files/mountnfs.sh index acc07797b..8d55c9175 100755 --- a/base/initscripts/centos/files/mountnfs.sh +++ b/base/initscripts/centos/files/mountnfs.sh @@ -15,14 +15,14 @@ if [ "$1" = "stop" ]; then fi # -# Run in a subshell because of I/O redirection. +# Run in a subshell because of I/O redirection. # test -f /etc/fstab && ( # -# Read through fstab line by line. If it is NFS, set the flag -# for mounting NFS filesystems. If any NFS partition is found and it -# not mounted with the nolock option, we start the rpcbind. +# Read through fstab line by line. If it is NFS, set the flag +# for mounting NFS filesystems. If any NFS partition is found and it +# not mounted with the nolock option, we start the rpcbind. # rpcbind=no mount_nfs=no @@ -31,41 +31,41 @@ mount_ncp=no mount_cifs=no while read device mountpt fstype options do - case "$device" in - ""|\#*) - continue - ;; - esac + case "$device" in + ""|\#*) + continue + ;; + esac - case "$options" in - *noauto*) - continue - ;; - esac + case "$options" in + *noauto*) + continue + ;; + esac - if test "$fstype" = nfs - then - mount_nfs=yes - case "$options" in - *nolock*) - ;; - *) - rpcbind=yes - ;; - esac - fi - if test "$fstype" = smbfs - then - mount_smb=yes - fi - if test "$fstype" = ncpfs - then - mount_ncp=yes - fi - if test "$fstype" = cifs - then - mount_cifs=yes - fi + if test "$fstype" = nfs + then + mount_nfs=yes + case "$options" in + *nolock*) + ;; + *) + rpcbind=yes + ;; + esac + fi + if test "$fstype" = smbfs + then + mount_smb=yes + fi + if test "$fstype" = ncpfs + then + mount_ncp=yes + fi + if test "$fstype" = cifs + then + mount_cifs=yes + fi done exec 0>&1 @@ -78,11 +78,11 @@ then then if test -x /usr/sbin/rpcbind then - service rpcbind status > /dev/null - if [ $? != 0 ]; then - echo -n "Starting rpcbind..." - start-stop-daemon --start --quiet --exec /usr/sbin/rpcbind - sleep 2 + service rpcbind status > /dev/null + if [ $? != 0 ]; then + echo -n "Starting rpcbind..." + start-stop-daemon --start --quiet --exec /usr/sbin/rpcbind + sleep 2 fi fi fi @@ -90,11 +90,11 @@ fi if test "$mount_nfs" = yes || test "$mount_smb" = yes || test "$mount_ncp" = yes || test "$mount_cifs" = yes then - echo "Mounting remote filesystems..." - test "$mount_nfs" = yes && mount -a -t nfs - test "$mount_smb" = yes && mount -a -t smbfs - test "$mount_ncp" = yes && mount -a -t ncpfs - test "$mount_cifs" = yes && mount -a -t cifs + echo "Mounting remote filesystems..." + test "$mount_nfs" = yes && mount -a -t nfs + test "$mount_smb" = yes && mount -a -t smbfs + test "$mount_ncp" = yes && mount -a -t ncpfs + test "$mount_cifs" = yes && mount -a -t cifs fi ) < /etc/fstab diff --git a/database/mariadb/centos/files/mysql-check-upgrade.sh b/database/mariadb/centos/files/mysql-check-upgrade.sh index 1bfd3bc6a..cd669bda8 100644 --- a/database/mariadb/centos/files/mysql-check-upgrade.sh +++ b/database/mariadb/centos/files/mysql-check-upgrade.sh @@ -27,9 +27,9 @@ if [ $version -ne $thisversion ] ; then cat <&2 The datadir located at $datadir needs to be upgraded using 'mysql_upgrade' tool. This can be done using the following steps: - 1. Back-up your data before with 'mysql_upgrade' - 2. Start the database daemon using 'service @DAEMON_NAME@ start' - 3. Run 'mysql_upgrade' with a database user that has sufficient privileges + 1. Back-up your data before with 'mysql_upgrade' + 2. Start the database daemon using 'service @DAEMON_NAME@ start' + 3. Run 'mysql_upgrade' with a database user that has sufficient privileges Read more about 'mysql_upgrade' usage at: https://mariadb.com/kb/en/mariadb/documentation/sql-commands/table-commands/mysql_upgrade/ diff --git a/database/mariadb/centos/files/mysql-prepare-db-dir.sh b/database/mariadb/centos/files/mysql-prepare-db-dir.sh index b47fa9b4a..96e899813 100644 --- a/database/mariadb/centos/files/mysql-prepare-db-dir.sh +++ b/database/mariadb/centos/files/mysql-prepare-db-dir.sh @@ -41,14 +41,14 @@ else fi myuser=`systemctl show -p User "${SERVICE_NAME}" | - sed 's/^User=//'` + sed 's/^User=//'` if [ x"$myuser" = x ] then myuser=mysql fi mygroup=`systemctl show -p Group "${SERVICE_NAME}" | - sed 's/^Group=//'` + sed 's/^Group=//'` if [ x"$mygroup" = x ] then mygroup=mysql diff --git a/database/mariadb/centos/files/mysql-scripts-common.sh b/database/mariadb/centos/files/mysql-scripts-common.sh index bf2888f2a..f77a8040b 100644 --- a/database/mariadb/centos/files/mysql-scripts-common.sh +++ b/database/mariadb/centos/files/mysql-scripts-common.sh @@ -11,18 +11,18 @@ export LC_ALL=C # We use my_print_defaults which prints all options from multiple files, # with the more specific ones later; hence take the last match. get_mysql_option(){ - if [ $# -ne 3 ] ; then - echo "get_mysql_option requires 3 arguments: section option default_value" - return - fi - sections="$1" - option_name="$2" - default_value="$3" - result=`@bindir@/my_print_defaults $sections | sed -n "s/^--${option_name}=//p" | tail -n 1` - if [ -z "$result" ]; then - # not found, use default - result="${default_value}" - fi + if [ $# -ne 3 ] ; then + echo "get_mysql_option requires 3 arguments: section option default_value" + return + fi + sections="$1" + option_name="$2" + default_value="$3" + result=`@bindir@/my_print_defaults $sections | sed -n "s/^--${option_name}=//p" | tail -n 1` + if [ -z "$result" ]; then + # not found, use default + result="${default_value}" + fi } # Defaults here had better match what mysqld_safe will default to diff --git a/database/mariadb/centos/files/mysql-wait-ready.sh b/database/mariadb/centos/files/mysql-wait-ready.sh index 2ed5fe1d3..6d95800aa 100644 --- a/database/mariadb/centos/files/mysql-wait-ready.sh +++ b/database/mariadb/centos/files/mysql-wait-ready.sh @@ -9,8 +9,8 @@ source "`dirname ${BASH_SOURCE[0]}`/mysql-scripts-common" # "After mysqld" won't be started until the database is really ready. if [ $# -ne 1 ] ; then - echo "You need to pass daemon pid as an argument for this script." - exit 20 + echo "You need to pass daemon pid as an argument for this script." + exit 20 fi # Service file passes us the daemon's PID (actually, mysqld_safe's PID) @@ -19,27 +19,27 @@ daemon_pid="$1" # Wait for the server to come up or for the mysqld process to disappear ret=0 while /bin/true; do - # Check process still exists - if ! [ -d "/proc/${daemon_pid}" ] ; then - ret=1 - break - fi - RESPONSE=`@bindir@/mysqladmin --no-defaults --socket="$socketfile" --user=UNKNOWN_MYSQL_USER ping 2>&1` - mret=$? - if [ $mret -eq 0 ] ; then - break - fi - # exit codes 1, 11 (EXIT_CANNOT_CONNECT_TO_SERVICE) are expected, - # anything else suggests a configuration error - if [ $mret -ne 1 -a $mret -ne 11 ]; then + # Check process still exists + if ! [ -d "/proc/${daemon_pid}" ] ; then + ret=1 + break + fi + RESPONSE=`@bindir@/mysqladmin --no-defaults --socket="$socketfile" --user=UNKNOWN_MYSQL_USER ping 2>&1` + mret=$? + if [ $mret -eq 0 ] ; then + break + fi + # exit codes 1, 11 (EXIT_CANNOT_CONNECT_TO_SERVICE) are expected, + # anything else suggests a configuration error + if [ $mret -ne 1 -a $mret -ne 11 ]; then echo "Cannot check for @NICE_PROJECT_NAME@ Daemon startup because of mysqladmin failure." >&2 - ret=$mret - break - fi - # "Access denied" also means the server is alive - echo "$RESPONSE" | grep -q "Access denied for user" && break + ret=$mret + break + fi + # "Access denied" also means the server is alive + echo "$RESPONSE" | grep -q "Access denied for user" && break - sleep 1 + sleep 1 done exit $ret diff --git a/database/mariadb/centos/files/mysql-wait-stop.sh b/database/mariadb/centos/files/mysql-wait-stop.sh index 62bde30ea..0af45fd51 100644 --- a/database/mariadb/centos/files/mysql-wait-stop.sh +++ b/database/mariadb/centos/files/mysql-wait-stop.sh @@ -13,24 +13,24 @@ TIMEOUT=60 SECONDS=0 if ! [ -f "$pidfile" ]; then - exit 0 + exit 0 fi MYSQLPID=`cat "$pidfile" 2>/dev/null` if [ -z "$MYSQLPID" ] ; then - exit 2 + exit 2 fi while /bin/true; do - # Check process still exists - if ! [ -d "/proc/${MYSQLPID}" ] ; then - break - fi - if [ $SECONDS -gt $TIMEOUT ] ; then - ret=3 - break - fi - sleep 1 + # Check process still exists + if ! [ -d "/proc/${MYSQLPID}" ] ; then + break + fi + if [ $SECONDS -gt $TIMEOUT ] ; then + ret=3 + break + fi + sleep 1 done exit $ret diff --git a/filesystem/nfscheck/files/nfscheck-init.sh b/filesystem/nfscheck/files/nfscheck-init.sh index ac2d86920..53b1fccea 100755 --- a/filesystem/nfscheck/files/nfscheck-init.sh +++ b/filesystem/nfscheck/files/nfscheck-init.sh @@ -22,7 +22,7 @@ DESC="nfscheck" DAEMON="/usr/bin/nfscheck" PIDFILE="/var/run/nfscheck.pid" -start() +start() { if [ -e $PIDFILE ]; then PIDDIR=/proc/$(cat $PIDFILE) @@ -47,7 +47,7 @@ start() fi } -stop() +stop() { echo -n "Stopping $DESC..." start-stop-daemon --stop --quiet --pidfile $PIDFILE diff --git a/networking/lldpd/centos/files/i40e-lldp-configure.sh b/networking/lldpd/centos/files/i40e-lldp-configure.sh index 2796c4304..431a405fa 100644 --- a/networking/lldpd/centos/files/i40e-lldp-configure.sh +++ b/networking/lldpd/centos/files/i40e-lldp-configure.sh @@ -28,7 +28,7 @@ LLDP_COMMAND=lldp function log() { local MSG="${PROGNAME}: $1" - logger -p notice "${MSG}" + logger -p notice "${MSG}" } function err() @@ -60,10 +60,10 @@ function configure_device() } function is_debugfs_mounted() { - if grep -qs "${DEBUGFS_PATH}" /proc/mounts; then - return 0 - fi - return 1 + if grep -qs "${DEBUGFS_PATH}" /proc/mounts; then + return 0 + fi + return 1 } function mount_debugfs() { @@ -82,7 +82,7 @@ function scan_devices() if is_debugfs_mounted; then DEBUGFS_MOUNTED="true" - fi + fi if [ ${DEBUGFS_MOUNTED} = "false" ]; then mount_debugfs @@ -120,7 +120,7 @@ function start() function stop() { scan_devices stop - return $? + return $? } function status() @@ -129,20 +129,20 @@ function status() } case "$1" in - start) + start) start ;; - stop) + stop) stop ;; - restart) + restart) stop start ;; - status) + status) status ;; - *) + *) echo "Usage: $0 {start|stop|restart|status}" exit 1 esac diff --git a/networking/mellanox/mlx4-config/files/mlx4-configure.sh b/networking/mellanox/mlx4-config/files/mlx4-configure.sh index 0953f0775..0005ce2e4 100644 --- a/networking/mellanox/mlx4-config/files/mlx4-configure.sh +++ b/networking/mellanox/mlx4-config/files/mlx4-configure.sh @@ -19,7 +19,7 @@ function log() { local MSG="${PROGNAME}: $1" if [ ${DEBUG} -ne 0 ]; then - echo "${MSG}" + echo "${MSG}" fi echo "${MSG}" >> /var/log/mlx4-configure.log } @@ -100,20 +100,20 @@ function status() } case "$1" in - start) + start) start ;; - stop) + stop) stop ;; - restart) + restart) stop start ;; - status) + status) status ;; - *) + *) echo "Usage: $0 {start|stop|restart|status}" exit 1 esac diff --git a/networking/mellanox/mlx4-config/files/mlx4_core_config.sh b/networking/mellanox/mlx4-config/files/mlx4_core_config.sh index 94816bff0..b1c0652d8 100644 --- a/networking/mellanox/mlx4-config/files/mlx4_core_config.sh +++ b/networking/mellanox/mlx4-config/files/mlx4_core_config.sh @@ -1,7 +1,7 @@ #!/bin/bash ################################################################################ # Copyright (c) 2015-2016 Wind River Systems, Inc. -# +# # SPDX-License-Identifier: Apache-2.0 # ################################################################################ diff --git a/tools/collector/scripts/collect_nfv_vim.sh b/tools/collector/scripts/collect_nfv_vim.sh index 4ccd309de..4b9928135 100644 --- a/tools/collector/scripts/collect_nfv_vim.sh +++ b/tools/collector/scripts/collect_nfv_vim.sh @@ -36,7 +36,7 @@ if [ "$nodetype" = "controller" ] ; then DATABASE_DIR=$(awk -F "=" '/database_dir/ {print $2}' /etc/nfv/vim/config.ini) SQLITE_DUMP="/usr/bin/sqlite3 ${DATABASE_DIR}/vim_db_v1 .dump" - + delimiter ${LOGFILE} "dump database" timeout 30 ${SQLITE_DUMP} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} fi diff --git a/tools/collector/scripts/collect_openstack.sh b/tools/collector/scripts/collect_openstack.sh index 743dd5d56..d4e986b62 100755 --- a/tools/collector/scripts/collect_openstack.sh +++ b/tools/collector/scripts/collect_openstack.sh @@ -51,7 +51,7 @@ num_exchanges=$(rabbitmqctl list_exchanges | wc -l); ((num_exchanges-=2)) num_connections=$(rabbitmqctl list_connections | wc -l); ((num_connections-=2)) num_channels=$(rabbitmqctl list_channels | wc -l); ((num_channels-=2)) arr=($(rabbitmqctl list_queues messages consumers memory | \ - awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) + awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]} printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" "queues" "bindings" "exchanges" "connections" "channels" "messages" "consumers" "memory" >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" $num_queues $num_bindings $num_exchanges $num_connections $num_channels $messages $consumers $memory >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} diff --git a/tools/collector/scripts/collect_psqldb.sh b/tools/collector/scripts/collect_psqldb.sh index 410aefa5c..f6613b60c 100755 --- a/tools/collector/scripts/collect_psqldb.sh +++ b/tools/collector/scripts/collect_psqldb.sh @@ -83,7 +83,7 @@ function database_dump() do if [ "$DB" != "keystone" -a "$DB" != "ceilometer" ] ; then echo "${hostname}: Dumping Database ..: ${DB_DIR}/$DB.$DB_EXT" - (cd ${DB_DIR} ; sudo -u postgres pg_dump $DB > $DB.$DB_EXT) + (cd ${DB_DIR} ; sudo -u postgres pg_dump $DB > $DB.$DB_EXT) fi done } @@ -107,7 +107,7 @@ if [ "$nodetype" = "controller" ] ; then pg_size_pretty(pg_database_size(pg_database.datname)) FROM pg_database ORDER BY pg_database_size DESC; - " >> ${LOGFILE} + " >> ${LOGFILE} # Number of postgres connections delimiter ${LOGFILE} "ps -C postgres -o cmd=" diff --git a/tools/engtools/hostdata-collectors/patch-engtools.sh b/tools/engtools/hostdata-collectors/patch-engtools.sh index f64614279..dbb80682e 100755 --- a/tools/engtools/hostdata-collectors/patch-engtools.sh +++ b/tools/engtools/hostdata-collectors/patch-engtools.sh @@ -3,15 +3,15 @@ # http://twiki.wrs.com/PBUeng/Patching if [ -z $MY_WORKSPACE ] || [ -z $MY_REPO ]; then - echo "Some dev environment variables are not set." - echo "Refer to http://wiki.wrs.com/PBUeng/CentOSBuildProcess for instructions." - exit 1 + echo "Some dev environment variables are not set." + echo "Refer to http://wiki.wrs.com/PBUeng/CentOSBuildProcess for instructions." + exit 1 fi ENGTOOLS=$(ls ${MY_WORKSPACE}/std/rpmbuild/RPMS/engtools*noarch.rpm 2>/dev/null) if [ $? -ne 0 ]; then - echo "Engtools RPM has not been built. Please run \"build-pkgs engtools\" first." - exit 1 + echo "Engtools RPM has not been built. Please run \"build-pkgs engtools\" first." + exit 1 fi source ${MY_REPO}/stx/middleware/recipes-common/build-info/release-info.inc diff --git a/tools/engtools/hostdata-collectors/scripts/ceph.sh b/tools/engtools/hostdata-collectors/scripts/ceph.sh index 40014406f..f05904ffc 100644 --- a/tools/engtools/hostdata-collectors/scripts/ceph.sh +++ b/tools/engtools/hostdata-collectors/scripts/ceph.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -16,17 +16,17 @@ OPT_USE_INTERVALS=1 # Print key ceph statistics function print_ceph() { - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME - cmd='ceph -s' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='ceph -s' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='ceph osd tree' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='ceph osd tree' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='ceph df detail' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='ceph df detail' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} } #------------------------------------------------------------------------------- @@ -49,8 +49,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_ceph - sleep ${INTERVAL_SEC} + print_ceph + sleep ${INTERVAL_SEC} done print_ceph LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh b/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh index e1f16239d..8fc7b8b1e 100644 --- a/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -ne 0 ]; then - ERRLOG "Require sudo/root access." - exit 1 + ERRLOG "Require sudo/root access." + exit 1 fi declare -a TOOLS @@ -34,23 +34,23 @@ LOG "kill processes brute force" pids=( $(pidof -x /usr/local/bin/collect-engtools.sh) ) if [ ${#pids[@]} -ne 0 ] then - LOG "killing: ${pids[@]}" - for pid in ${pids[@]} - do + LOG "killing: ${pids[@]}" + for pid in ${pids[@]} + do LOG "kill: [ ${pid} ] " pkill -KILL -P ${pid} kill -9 ${pid} - done - pkill -KILL iostat - pkill -KILL top + done + pkill -KILL iostat + pkill -KILL top else - LOG "no pids found" + LOG "no pids found" fi LOG "remove pidfiles" for TOOL in "${TOOLS[@]}" do - rm -f -v /var/run/${TOOL}.pid + rm -f -v /var/run/${TOOL}.pid done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh b/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh index 7c1887a13..5e3c06430 100644 --- a/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh @@ -50,26 +50,26 @@ declare tgt_used_bytes # do_parallel_commands - launch parallel tools with separate output files function do_parallel_commands() { - parallel_outfiles=() - for elem in "${tlist[@]}" - do - tool=""; period=""; repeat=""; interval="" - my_hash="elem[*]" - local ${!my_hash} - if [ ! -z "${name}" ]; then - fname="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${name}" - parallel_outfiles+=( $fname ) - LOG "collecting ${tool}, ${interval} second intervals, to: ${fname}" - if [ ! -z "${period}" ]; then - ${STDBUF} ${tool} -p ${period} -i ${interval} > ${fname} 2>/dev/null & - elif [ ! -z "${repeat}" ]; then - ${STDBUF} ${tool} --repeat=${repeat} --delay=${interval} > ${fname} 2>/dev/null & - fi - else - # run without file output (eg., ticker) - ${STDBUF} ${tool} -p ${period} -i ${interval} 2>/dev/null & - fi - done + parallel_outfiles=() + for elem in "${tlist[@]}" + do + tool=""; period=""; repeat=""; interval="" + my_hash="elem[*]" + local ${!my_hash} + if [ ! -z "${name}" ]; then + fname="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${name}" + parallel_outfiles+=( $fname ) + LOG "collecting ${tool}, ${interval} second intervals, to: ${fname}" + if [ ! -z "${period}" ]; then + ${STDBUF} ${tool} -p ${period} -i ${interval} > ${fname} 2>/dev/null & + elif [ ! -z "${repeat}" ]; then + ${STDBUF} ${tool} --repeat=${repeat} --delay=${interval} > ${fname} 2>/dev/null & + fi + else + # run without file output (eg., ticker) + ${STDBUF} ${tool} -p ${period} -i ${interval} 2>/dev/null & + fi + done } # get_current_avail_usage() - get output destination file-system usage and @@ -77,10 +77,10 @@ function do_parallel_commands() # - updates: df_size_bytes, df_avail_bytes, du_used_bytes function get_current_avail_usage() { - local -a df_arr_bytes=( $(df -P --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==2 {print $2, $4}') ) - df_size_bytes=${df_arr_bytes[0]} - df_avail_bytes=${df_arr_bytes[1]} - du_used_bytes=$(du --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==1 {print $1}') + local -a df_arr_bytes=( $(df -P --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==2 {print $2, $4}') ) + df_size_bytes=${df_arr_bytes[0]} + df_avail_bytes=${df_arr_bytes[1]} + du_used_bytes=$(du --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==1 {print $1}') } # purge_oldest_files() - remove oldest files based on file-system available space, @@ -88,57 +88,57 @@ function get_current_avail_usage() function purge_oldest_files() { # get current file-system usage - get_current_avail_usage - msg=$(printf "avail %d MB, headroom %d MB; used %d MB, max %d MB" \ + get_current_avail_usage + msg=$(printf "avail %d MB, headroom %d MB; used %d MB, max %d MB" \ $[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] \ $[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024]) - LOG "usage: ${msg}" + LOG "usage: ${msg}" - if [[ $df_avail_bytes -lt $tgt_avail_bytes ]] || \ - [[ $du_used_bytes -gt $tgt_used_bytes ]]; then - # wait for compression to complete - wait + if [[ $df_avail_bytes -lt $tgt_avail_bytes ]] || \ + [[ $du_used_bytes -gt $tgt_used_bytes ]]; then + # wait for compression to complete + wait - get_current_avail_usage - if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then - msg=$(printf "purge: avail %d MB < target %d MB" \ - $[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] ) - LOG "purge: ${msg}" + get_current_avail_usage + if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then + msg=$(printf "purge: avail %d MB < target %d MB" \ + $[$df_avail_bytes/1024/1024] $[$tgt_avail_bytes/1024/1024] ) + LOG "purge: ${msg}" + fi + if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then + msg=$(printf "purge: used %d MB > target %d MB" \ + $[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024] ) + LOG "purge: ${msg}" + fi + else + return fi - if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then - msg=$(printf "purge: used %d MB > target %d MB" \ - $[$du_used_bytes/1024/1024] $[$tgt_used_bytes/1024/1024] ) - LOG "purge: ${msg}" - fi - else - return - fi # remove files in oldest time sorted order until we meet usage targets, # incrementally updating usage as we remve files - for file in $( ls -rt ${TOOL_DEST_DIR}/${HOSTNAME}_* 2>/dev/null ) - do - if [[ $df_avail_bytes -ge $tgt_avail_bytes ]] && \ - [[ $du_used_bytes -le $tgt_used_bytes ]]; then - break - fi + for file in $( ls -rt ${TOOL_DEST_DIR}/${HOSTNAME}_* 2>/dev/null ) + do + if [[ $df_avail_bytes -ge $tgt_avail_bytes ]] && \ + [[ $du_used_bytes -le $tgt_used_bytes ]]; then + break + fi - if [ ${OPT_DEBUG} -eq 1 ]; then - msg="purge: file=$file" - if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then - msg="${msg}, < AVAIL" - fi - if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then - msg="${msg}, > MAXUSAGE" - fi - LOG "${msg}" - fi + if [ ${OPT_DEBUG} -eq 1 ]; then + msg="purge: file=$file" + if [[ $df_avail_bytes -lt $tgt_avail_bytes ]]; then + msg="${msg}, < AVAIL" + fi + if [[ $du_used_bytes -gt $tgt_used_bytes ]]; then + msg="${msg}, > MAXUSAGE" + fi + LOG "${msg}" + fi - sz_bytes=$(stat --printf="%s" $file) - ((df_avail_bytes += sz_bytes)) - ((du_used_bytes -= sz_bytes)) - rm -fv ${file} - done + sz_bytes=$(stat --printf="%s" $file) + ((df_avail_bytes += sz_bytes)) + ((du_used_bytes -= sz_bytes)) + rm -fv ${file} + done } #------------------------------------------------------------------------------- @@ -160,23 +160,23 @@ CPULIST="" # Affine tools to NOVA pinned cores (i.e., non-cpu 0) # - remove interference with cpu 0 if [ "${AFFINE_PINNED}" -eq 1 ]; then - NOVA_CONF=/etc/nova/compute_extend.conf - if [ -f "${NOVA_CONF}" ]; then + NOVA_CONF=/etc/nova/compute_extend.conf + if [ -f "${NOVA_CONF}" ]; then source "${NOVA_CONF}" CPULIST=${compute_pinned_cpulist} - else + else CPULIST="" - fi + fi fi set_affinity ${CPULIST} # Define output directory if [[ "${HOSTNAME}" =~ "controller-" ]]; then - TOOL_DEST_DIR=/scratch/syseng_data/${HOSTNAME} + TOOL_DEST_DIR=/scratch/syseng_data/${HOSTNAME} elif [[ "${HOSTNAME}" =~ "compute-" ]]; then - TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} + TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} else - TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} + TOOL_DEST_DIR=/tmp/syseng_data/${HOSTNAME} fi mkdir -p ${TOOL_DEST_DIR} @@ -186,7 +186,7 @@ DAEMON_OUT="${TOOL_DEST_DIR}/${HOSTNAME}_${timestamp}_${TOOLNAME}.log" # Redirect stdout and append to log if not connected to TTY if test ! -t 1 ; then - exec 1>> ${DAEMON_OUT} + exec 1>> ${DAEMON_OUT} fi # Get current availability and usage @@ -199,9 +199,9 @@ tgt_used_bytes=$[$PURGE_MAXUSAGE_MB*1024*1024] # Set granularity based on duration if [ $PERIOD_MIN -le 30 ]; then - GRAN_MIN=5 + GRAN_MIN=5 else - GRAN_MIN=60 + GRAN_MIN=60 fi # Adjust repeats and intervals based on GRAN_MIN granularity @@ -209,7 +209,7 @@ PERIOD_MIN=$[($PERIOD_MIN+(GRAN_MIN-1))/GRAN_MIN*GRAN_MIN] ((REPEATS = PERIOD_MIN/GRAN_MIN)) GRAN_MIN_IN_SEC=$[$GRAN_MIN*60] if [ ${INTERVAL_SEC} -gt ${GRAN_MIN_IN_SEC} ]; then - INTERVAL_SEC=${GRAN_MIN_IN_SEC} + INTERVAL_SEC=${GRAN_MIN_IN_SEC} fi # Define tools and options @@ -231,7 +231,7 @@ if [ ${OPT_SOAK} -eq 1 ]; then ceph_interval=${DUR_15MIN_IN_SEC} diskstats_interval=${DUR_15MIN_IN_SEC} memstats_interval=${DUR_15MIN_IN_SEC} - filestats_interval=${DUR_15MIN_IN_SEC} + filestats_interval=${DUR_15MIN_IN_SEC} elif [ ${OPT_SOAK} -eq 2 ]; then # Assume much shorter collection (eg, < hours) schedtop_interval=2 # i.e., 2 second interval @@ -243,7 +243,7 @@ elif [ ${OPT_SOAK} -eq 2 ]; then ceph_interval=${DUR_5MIN_IN_SEC} diskstats_interval=${DUR_5MIN_IN_SEC} memstats_interval=${DUR_5MIN_IN_SEC} - filestats_interval=${DUR_5MIN_IN_SEC} + filestats_interval=${DUR_5MIN_IN_SEC} else # Assume shorter collection (eg, < a few hours) schedtop_interval=5 # i.e., 5 second interval @@ -255,7 +255,7 @@ else ceph_interval=${DUR_5MIN_IN_SEC} diskstats_interval=${DUR_5MIN_IN_SEC} memstats_interval=${DUR_5MIN_IN_SEC} - filestats_interval=${DUR_5MIN_IN_SEC} + filestats_interval=${DUR_5MIN_IN_SEC} fi schedtop_repeat=$[ $PERIOD_MIN * 60 / $schedtop_interval ] occtop_repeat=$[ $PERIOD_MIN * 60 / $occtop_interval ] @@ -274,29 +274,29 @@ LBINDIR=/usr/local/bin declare -a tlist if [[ ${ENABLE_STATIC_COLLECTION} == "Y" ]] || [[ ${ENABLE_STATIC_COLLECTION} == "y" ]]; then - tlist+=( "tool=${LBINDIR}/top.sh name=top period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) - tlist+=( "tool=${LBINDIR}/iostat.sh name=iostat period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) - tlist+=( "tool=${LBINDIR}/netstats.sh name=netstats period=${PERIOD_MIN} interval=${netstats_interval}" ) - tlist+=( "tool=${BINDIR}/occtop name=occtop repeat=${occtop_repeat} interval=${occtop_interval}" ) - tlist+=( "tool=${BINDIR}/memtop name=memtop repeat=${memtop_repeat} interval=${memtop_interval}" ) - tlist+=( "tool=${BINDIR}/schedtop name=schedtop repeat=${schedtop_repeat} interval=${schedtop_interval}" ) - tlist+=( "tool=${LBINDIR}/diskstats.sh name=diskstats period=${PERIOD_MIN} interval=${diskstats_interval}" ) - tlist+=( "tool=${LBINDIR}/memstats.sh name=memstats period=${PERIOD_MIN} interval=${memstats_interval}" ) - tlist+=( "tool=${LBINDIR}/filestats.sh name=filestats period=${PERIOD_MIN} interval=${filestats_interval}" ) - if [[ "${HOSTNAME}" =~ "controller-" ]]; then - tlist+=( "tool=${LBINDIR}/ceph.sh name=ceph period=${PERIOD_MIN} interval=${ceph_interval}" ) - tlist+=( "tool=${LBINDIR}/postgres.sh name=postgres period=${PERIOD_MIN} interval=${postgres_interval}" ) - tlist+=( "tool=${LBINDIR}/rabbitmq.sh name=rabbitmq period=${PERIOD_MIN} interval=${rabbitmq_interval}" ) - elif [[ "${HOSTNAME}" =~ "compute-" ]]; then - tlist+=( "tool=${LBINDIR}/vswitch.sh name=vswitch period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) - fi + tlist+=( "tool=${LBINDIR}/top.sh name=top period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + tlist+=( "tool=${LBINDIR}/iostat.sh name=iostat period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + tlist+=( "tool=${LBINDIR}/netstats.sh name=netstats period=${PERIOD_MIN} interval=${netstats_interval}" ) + tlist+=( "tool=${BINDIR}/occtop name=occtop repeat=${occtop_repeat} interval=${occtop_interval}" ) + tlist+=( "tool=${BINDIR}/memtop name=memtop repeat=${memtop_repeat} interval=${memtop_interval}" ) + tlist+=( "tool=${BINDIR}/schedtop name=schedtop repeat=${schedtop_repeat} interval=${schedtop_interval}" ) + tlist+=( "tool=${LBINDIR}/diskstats.sh name=diskstats period=${PERIOD_MIN} interval=${diskstats_interval}" ) + tlist+=( "tool=${LBINDIR}/memstats.sh name=memstats period=${PERIOD_MIN} interval=${memstats_interval}" ) + tlist+=( "tool=${LBINDIR}/filestats.sh name=filestats period=${PERIOD_MIN} interval=${filestats_interval}" ) + if [[ "${HOSTNAME}" =~ "controller-" ]]; then + tlist+=( "tool=${LBINDIR}/ceph.sh name=ceph period=${PERIOD_MIN} interval=${ceph_interval}" ) + tlist+=( "tool=${LBINDIR}/postgres.sh name=postgres period=${PERIOD_MIN} interval=${postgres_interval}" ) + tlist+=( "tool=${LBINDIR}/rabbitmq.sh name=rabbitmq period=${PERIOD_MIN} interval=${rabbitmq_interval}" ) + elif [[ "${HOSTNAME}" =~ "compute-" ]]; then + tlist+=( "tool=${LBINDIR}/vswitch.sh name=vswitch period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + fi # ticker - shows progress on the screen - tlist+=( "tool=${LBINDIR}/ticker.sh name= period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) + tlist+=( "tool=${LBINDIR}/ticker.sh name= period=${PERIOD_MIN} interval=${DUR_1MIN_IN_SEC}" ) fi if [[ ${ENABLE_LIVE_STREAM} == "Y" ]] || [[ ${ENABLE_LIVE_STREAM} == "y" ]]; then - ${TOOLBIN}/live_stream.py & + ${TOOLBIN}/live_stream.py & fi #------------------------------------------------------------------------------- @@ -306,32 +306,32 @@ OPT_DEBUG=0 REP=0 if [ ${#tlist[@]} -ne 0 ]; then - # Static stats collection is turned on - while [[ ${TOOL_USR1_SIGNAL} -eq 0 ]] && + # Static stats collection is turned on + while [[ ${TOOL_USR1_SIGNAL} -eq 0 ]] && [[ ${OPT_FOREVER} -eq 1 || ${REP} -lt ${REPEATS} ]] - do - # increment loop counter - ((REP++)) + do + # increment loop counter + ((REP++)) - # purge oldest files - purge_oldest_files + # purge oldest files + purge_oldest_files - # define filename timestamp - timestamp=$( date +"%Y-%0m-%0e_%H%M" ) + # define filename timestamp + timestamp=$( date +"%Y-%0m-%0e_%H%M" ) - # collect tools in parallel to separate output files - LOG "collecting ${TOOLNAME} at ${timestamp} for ${PERIOD_MIN} mins, repeat=${REP}" - do_parallel_commands - wait + # collect tools in parallel to separate output files + LOG "collecting ${TOOLNAME} at ${timestamp} for ${PERIOD_MIN} mins, repeat=${REP}" + do_parallel_commands + wait + + # Compress latest increment + LOG "compressing: ${parallel_outfiles[@]}" + ${CMD_IDLE} bzip2 -q -f ${parallel_outfiles[@]} 2>/dev/null & + done - # Compress latest increment - LOG "compressing: ${parallel_outfiles[@]}" - ${CMD_IDLE} bzip2 -q -f ${parallel_outfiles[@]} 2>/dev/null & - done - # Wait for the compression to complete - wait - tools_cleanup 0 + wait + tools_cleanup 0 fi # Should wait here in case live stats streaming is turned on. diff --git a/tools/engtools/hostdata-collectors/scripts/diskstats.sh b/tools/engtools/hostdata-collectors/scripts/diskstats.sh index 376dbf185..d00d777ba 100644 --- a/tools/engtools/hostdata-collectors/scripts/diskstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/diskstats.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -16,76 +16,76 @@ OPT_USE_INTERVALS=1 # Print disk summary function print_disk() { - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME # NOTES: # --total (grand-total) is a new option, but don't necessarily want to add tmpfs - # or dummy filesystems. + # or dummy filesystems. # - use -H to print in SI (eg, GB, vs GiB) # - can use -a to print all filesystems including dummy filesystems, but then # there can be double-counting: - print_separator - cmd='df -h -H -T --local -t ext2 -t ext3 -t ext4 -t xfs --total' - ${ECHO} "Disk space usage ext2,ext3,ext4,xfs,tmpfs (SI):" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='df -h -H -T --local -t ext2 -t ext3 -t ext4 -t xfs --total' + ${ECHO} "Disk space usage ext2,ext3,ext4,xfs,tmpfs (SI):" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='df -h -H -T --local -i -t ext2 -t ext3 -t ext4 -t xfs --total' - ${ECHO} "Disk inodes usage ext2,ext3,ext4,xfs,tmpfs (SI):" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='df -h -H -T --local -i -t ext2 -t ext3 -t ext4 -t xfs --total' + ${ECHO} "Disk inodes usage ext2,ext3,ext4,xfs,tmpfs (SI):" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='drbd-overview' - ${ECHO} "drbd disk usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='drbd-overview' + ${ECHO} "drbd disk usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='lvs' - ${ECHO} "logical volumes usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='lvs' + ${ECHO} "logical volumes usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='pvs' - ${ECHO} "physical volumes usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='pvs' + ${ECHO} "physical volumes usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - print_separator - cmd='vgs' - ${ECHO} "volume groups usage and status:" - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + print_separator + cmd='vgs' + ${ECHO} "volume groups usage and status:" + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} } # Print disk static summary function print_disk_static() { - print_separator - cmd='cat /proc/scsi/scsi' - ${ECHO} "Attached devices: ${cmd}" - ${cmd} - ${ECHO} + print_separator + cmd='cat /proc/scsi/scsi' + ${ECHO} "Attached devices: ${cmd}" + ${cmd} + ${ECHO} # fdisk - requires sudo/root - print_separator - cmd='fdisk -l' - if [ $UID -eq 0 ]; then - ${ECHO} "List disk devices: ${cmd}" - ${cmd} - else - WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" - fi - ${ECHO} + print_separator + cmd='fdisk -l' + if [ $UID -eq 0 ]; then + ${ECHO} "List disk devices: ${cmd}" + ${cmd} + else + WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" + fi + ${ECHO} # parted - requires sudo/root - print_separator - cmd='parted -l' - if [ $UID -eq 0 ]; then - ${ECHO} "List disk devices: ${cmd}" - ${cmd} - else - WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" - fi - ${ECHO} + print_separator + cmd='parted -l' + if [ $UID -eq 0 ]; then + ${ECHO} "List disk devices: ${cmd}" + ${cmd} + else + WARNLOG "Skipping cmd=${cmd}, root/sudo passwd required" + fi + ${ECHO} } #------------------------------------------------------------------------------- @@ -111,8 +111,8 @@ print_disk_static for ((rep=1; rep <= REPEATS ; rep++)) do - print_disk - sleep ${INTERVAL_SEC} + print_disk + sleep ${INTERVAL_SEC} done print_disk LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/engtools_util.sh b/tools/engtools/hostdata-collectors/scripts/engtools_util.sh index 311ccd2bf..4ac66a9ae 100644 --- a/tools/engtools/hostdata-collectors/scripts/engtools_util.sh +++ b/tools/engtools/hostdata-collectors/scripts/engtools_util.sh @@ -7,7 +7,7 @@ TOOL_USR1_SIGNAL=0 TOOL_USR2_SIGNAL=0 TOOL_TTY=0 if tty 1>/dev/null ; then - TOOL_TTY=1 + TOOL_TTY=1 fi # [ JGAULD : SHOULD RENAME TO TOOL_X ] @@ -19,145 +19,145 @@ CPULIST=0 # Include lsb functions if [ -d /lib/lsb ]; then -. /lib/lsb/init-functions + . /lib/lsb/init-functions else -. /etc/init.d/functions + . /etc/init.d/functions fi # Lightweight replacement for pidofproc -p function check_pidfile () { - local pidfile pid + local pidfile pid - OPTIND=1 - while getopts p: opt ; do + OPTIND=1 + while getopts p: opt ; do case "$opt" in - p) + p) pidfile="$OPTARG" ;; esac - done - shift $(($OPTIND - 1)) + done + shift $(($OPTIND - 1)) - read pid < "${pidfile}" - if [ -n "${pid:-}" ]; then - if $(kill -0 "${pid:-}" 2> /dev/null); then - echo "$pid" - return 0 - elif ps "${pid:-}" >/dev/null 2>&1; then - echo "$pid" - return 0 # program is running, but not owned by this user - else - return 1 # program is dead and /var/run pid file exists + read pid < "${pidfile}" + if [ -n "${pid:-}" ]; then + if $(kill -0 "${pid:-}" 2> /dev/null); then + echo "$pid" + return 0 + elif ps "${pid:-}" >/dev/null 2>&1; then + echo "$pid" + return 0 # program is running, but not owned by this user + else + return 1 # program is dead and /var/run pid file exists + fi fi - fi } # tools_init - initialize tool resources function tools_init () { - local rc=0 - local error=0 - TOOLNAME=$(basename $0) + local rc=0 + local error=0 + TOOLNAME=$(basename $0) # Check for sufficient priviledges - if [ $UID -ne 0 ]; then - ERRLOG "${NAME} requires sudo/root access." - return 1 - fi + if [ $UID -ne 0 ]; then + ERRLOG "${NAME} requires sudo/root access." + return 1 + fi # Check for essential binaries - ECHO=$(which echo 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ECHO=echo # use bash built-in echo - ${ECHO} "FATAL, 'echo' not found, rc=$rc"; - error=$rc - fi - DATE=$(which date 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ${ECHO} "FATAL, 'date' not found, rc=$rc"; - error=$rc - fi + ECHO=$(which echo 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ECHO=echo # use bash built-in echo + ${ECHO} "FATAL, 'echo' not found, rc=$rc"; + error=$rc + fi + DATE=$(which date 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ${ECHO} "FATAL, 'date' not found, rc=$rc"; + error=$rc + fi # Check for standard linux binaries, at least can use LOG functions now # - these are used in tools_header - CAT=$(which cat 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'cat' not found, rc=$rc"; - error=$rc - fi + CAT=$(which cat 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'cat' not found, rc=$rc"; + error=$rc + fi - ARCH=$(which arch 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'arch' not found, rc=$rc"; - error=$rc - fi + ARCH=$(which arch 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'arch' not found, rc=$rc"; + error=$rc + fi - SED=$(which sed 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'sed' not found, rc=$rc"; - error=$rc - fi + SED=$(which sed 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'sed' not found, rc=$rc"; + error=$rc + fi - GREP=$(which grep 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'grep' not found, rc=$rc"; - error=$rc - fi + GREP=$(which grep 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'grep' not found, rc=$rc"; + error=$rc + fi - WC=$(which wc 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'wc' not found, rc=$rc"; - error=$rc - fi + WC=$(which wc 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'wc' not found, rc=$rc"; + error=$rc + fi - UNAME=$(which uname 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'uname' not found, rc=$rc"; - error=$rc - fi + UNAME=$(which uname 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'uname' not found, rc=$rc"; + error=$rc + fi - SORT=$(which sort 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'sort' not found, rc=$rc"; - error=$rc - fi + SORT=$(which sort 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'sort' not found, rc=$rc"; + error=$rc + fi - TR=$(which tr 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'tr' not found, rc=$rc"; - error=$rc - fi + TR=$(which tr 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'tr' not found, rc=$rc"; + error=$rc + fi - AWK=$(which awk 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'awk' not found, rc=$rc"; - error=$rc - fi + AWK=$(which awk 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'awk' not found, rc=$rc"; + error=$rc + fi - PKILL=$(which pkill 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'pkill' not found, rc=$rc"; - error=$rc - fi + PKILL=$(which pkill 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'pkill' not found, rc=$rc"; + error=$rc + fi - LS=$(which ls 2>/dev/null) - rc=$? - if [ $rc -ne 0 ]; then - ERRLOG "'ls' not found, rc=$rc"; - error=$rc - fi + LS=$(which ls 2>/dev/null) + rc=$? + if [ $rc -ne 0 ]; then + ERRLOG "'ls' not found, rc=$rc"; + error=$rc + fi # The following block is needed for LSB systems such as Windriver Linux. # The utility is not available on CentOS so comment it out. @@ -170,91 +170,91 @@ function tools_init () # Let parent program decide what to do with the errors, # give ominous warning - if [ $error -eq 1 ]; then - WARNLOG "possibly cannot continue, missing linux binaries" - fi + if [ $error -eq 1 ]; then + WARNLOG "possibly cannot continue, missing linux binaries" + fi # Check if tool was previously running - if [ -e ${PIDFILE} ]; then + if [ -e ${PIDFILE} ]; then # [ JGAULD - remove pidofproc() / LSB compatibility issue ] if check_pidfile -p "${PIDFILE}" >/dev/null; then - ERRLOG "${PIDFILE} exists and ${TOOLNAME} is running" - return 1 + ERRLOG "${PIDFILE} exists and ${TOOLNAME} is running" + return 1 else # remove pid file - WARNLOG "${PIDFILE} exists but ${TOOLNAME} is not running; cleaning up" - rm -f ${PIDFILE} + WARNLOG "${PIDFILE} exists but ${TOOLNAME} is not running; cleaning up" + rm -f ${PIDFILE} + fi fi - fi # Create pid file - echo $$ > ${PIDFILE} + echo $$ > ${PIDFILE} # Setup trap handler - these signals trigger child shutdown and cleanup - trap tools_exit_handler INT HUP TERM EXIT - trap tools_usr1_handler USR1 - trap tools_usr2_handler USR2 + trap tools_exit_handler INT HUP TERM EXIT + trap tools_usr1_handler USR1 + trap tools_usr2_handler USR2 - return ${rc} + return ${rc} } # tools_cleanup() - terminate child processes function tools_cleanup() { # restore signal handling to default behaviour - trap - INT HUP TERM EXIT - trap - USR1 USR2 + trap - INT HUP TERM EXIT + trap - USR1 USR2 - local VERBOSE_OPT='' - if [ "$1" -ne "0" ]; then - LOG "cleanup invoked with code: $1" - if [ ${TOOL_DEBUG} -ne 0 ]; then - VERBOSE_OPT='-v' + local VERBOSE_OPT='' + if [ "$1" -ne "0" ]; then + LOG "cleanup invoked with code: $1" + if [ ${TOOL_DEBUG} -ne 0 ]; then + VERBOSE_OPT='-v' + fi fi - fi # stop all processes launched from this process - pkill -TERM -P $$ - if [ "$1" -ne "0" ]; then - sleep 1 - fi + pkill -TERM -P $$ + if [ "$1" -ne "0" ]; then + sleep 1 + fi # OK, if the above didn't work, use force - pkill -KILL -P $$ + pkill -KILL -P $$ # remove pid file - if [ -e ${PIDFILE} ]; then - rm -f ${VERBOSE_OPT} ${PIDFILE} - fi - exit $1 + if [ -e ${PIDFILE} ]; then + rm -f ${VERBOSE_OPT} ${PIDFILE} + fi + exit $1 } # tools_exit_handler() - exit handler routine function tools_exit_handler() { - TOOL_EXIT_SIGNAL=1 - tools_cleanup 128 + TOOL_EXIT_SIGNAL=1 + tools_cleanup 128 } # tools_usr1_handler() - USR1 handler routine function tools_usr1_handler() { - TOOL_USR1_SIGNAL=1 - LOG "caught USR1" + TOOL_USR1_SIGNAL=1 + LOG "caught USR1" } # tools_usr2_handler() - USR2 handler routine function tools_usr2_handler() { - TOOL_USR2_SIGNAL=1 - LOG "caught USR1" + TOOL_USR2_SIGNAL=1 + LOG "caught USR1" } # LOG(), WARNLOG(), ERRLOG() - simple print log functions (not logger) function LOG () { - local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) - echo "${tstamp_H} ${HOSTNAME} $0($$): $@"; + local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) + echo "${tstamp_H} ${HOSTNAME} $0($$): $@"; } function LOG_NOCR () { - local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) - echo -n "${tstamp_H} ${HOSTNAME} $0($$): $@"; + local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) + echo -n "${tstamp_H} ${HOSTNAME} $0($$): $@"; } function WARNLOG () { LOG "WARN $@"; } function ERRLOG () { LOG "ERROR $@"; } @@ -262,76 +262,76 @@ function ERRLOG () { LOG "ERROR $@"; } # TOOL_HIRES_TIME() - easily parsed date/timestamp and hi-resolution uptime function TOOL_HIRES_TIME() { - echo "time: " $( ${DATE} +"%a %F %H:%M:%S.%N %Z %z" ) "uptime: " $( cat /proc/uptime ) + echo "time: " $( ${DATE} +"%a %F %H:%M:%S.%N %Z %z" ) "uptime: " $( cat /proc/uptime ) } # set_affinity() - set affinity for current script if a a CPULIST is defined function set_affinity() { - local CPULIST=$1 - if [ -z "${CPULIST}" ]; then - return - fi + local CPULIST=$1 + if [ -z "${CPULIST}" ]; then + return + fi # Set cpu affinity for current program - local TASKSET=$(which taskset 2>/dev/null) - if [ -x "${TASKSET}" ]; then - ${TASKSET} -pc ${CPULIST} $$ 2>/dev/null - fi + local TASKSET=$(which taskset 2>/dev/null) + if [ -x "${TASKSET}" ]; then + ${TASKSET} -pc ${CPULIST} $$ 2>/dev/null + fi } # cmd_idle_priority() - command to set nice + ionice function cmd_idle_priority() { - local NICE="" - local IONICE="" + local NICE="" + local IONICE="" - NICE=$( which nice 2>/dev/null ) - if [ $? -eq 0 ]; then - NICE="${NICE} -n 19" - else - NICE="" - fi - IONICE=$( which ionice 2>/dev/null ) - if [ $? -eq 0 ]; then - IONICE="${IONICE} -c 3" - else - IONICE="" - fi - echo "${NICE} ${IONICE}" + NICE=$( which nice 2>/dev/null ) + if [ $? -eq 0 ]; then + NICE="${NICE} -n 19" + else + NICE="" + fi + IONICE=$( which ionice 2>/dev/null ) + if [ $? -eq 0 ]; then + IONICE="${IONICE} -c 3" + else + IONICE="" + fi + echo "${NICE} ${IONICE}" } # print_separator() - print a horizontal separation line '\u002d' is '-' function print_separator () { - printf '\u002d%.s' {1..80} - printf '\n' + printf '\u002d%.s' {1..80} + printf '\n' } # tools_header() - print out common GenWare tools header function tools_header() { - local TOOLNAME=$(basename $0) + local TOOLNAME=$(basename $0) # Get timestamp #local tstamp=$( date +"%Y-%0m-%0e %H:%M:%S" 2>/dev/null ) - local tstamp=$( date --rfc-3339=ns | cut -c1-23 2>/dev/null ) + local tstamp=$( date --rfc-3339=ns | cut -c1-23 2>/dev/null ) # Linux Generic - local UPTIME=/proc/uptime + local UPTIME=/proc/uptime # Get number of online cpus - local CPUINFO=/proc/cpuinfo - local online_cpus=$( cat ${CPUINFO} | grep -i ^processor | wc -l 2>/dev/null ) + local CPUINFO=/proc/cpuinfo + local online_cpus=$( cat ${CPUINFO} | grep -i ^processor | wc -l 2>/dev/null ) # Get load average, run-queue size, and number of threads - local LOADAVG=/proc/loadavg - local LDAVG=( `cat ${LOADAVG} | sed -e 's#[/]# #g' 2>/dev/null` ) + local LOADAVG=/proc/loadavg + local LDAVG=( `cat ${LOADAVG} | sed -e 's#[/]# #g' 2>/dev/null` ) # Get current architecture - local arch=$( uname -m ) + local arch=$( uname -m ) # Determine processor name (there are many different formats... *sigh* ) # - build up info from multiple lines - local processor='unk' - local NAME=$( cat ${CPUINFO} | grep \ + local processor='unk' + local NAME=$( cat ${CPUINFO} | grep \ -e '^cpu\W\W:' \ -e ^'cpu model' \ -e ^'model name' \ @@ -340,53 +340,53 @@ function tools_header() { -e ^[Mm]achine | \ sort -u | awk 'BEGIN{FS=":";} {print $2;}' | \ tr '\n' ' ' | tr -s [:blank:] 2>/dev/null ) - if [ ! -z "${NAME}" ]; then - processor=${NAME} - fi + if [ ! -z "${NAME}" ]; then + processor=${NAME} + fi # Determine processor speed (abort grep after first match) - local speed='unk' - local BOGO=$( cat ${CPUINFO} | grep -m1 -e ^BogoMIPS -e ^bogomips | \ + local speed='unk' + local BOGO=$( cat ${CPUINFO} | grep -m1 -e ^BogoMIPS -e ^bogomips | \ awk 'BEGIN{FS=":";} {printf "%.1f", $2;}' 2>/dev/null ) - local MHZ=$( cat ${CPUINFO} | grep -m1 -e ^'cpu MHz' -e ^clock | \ + local MHZ=$( cat ${CPUINFO} | grep -m1 -e ^'cpu MHz' -e ^clock | \ awk 'BEGIN{FS=":";} {printf "%.1f", $2;}' 2>/dev/null ) - local MHZ2=$( cat ${CPUINFO} | grep -m1 -e ^Cpu0ClkTck -e ^'cycle frequency' | \ + local MHZ2=$( cat ${CPUINFO} | grep -m1 -e ^Cpu0ClkTck -e ^'cycle frequency' | \ awk 'BEGIN{FS=":";} {printf "%.1f", $2/1.0E6;}' 2>/dev/null ) - if [ ! -z "${MHZ}" ]; then - speed=${MHZ} - elif [ ! -z "${MHZ2}" ]; then - speed=${MHZ2} - elif [ ! -z ${BOGO} ]; then - speed=${BOGO} - fi + if [ ! -z "${MHZ}" ]; then + speed=${MHZ} + elif [ ! -z "${MHZ2}" ]; then + speed=${MHZ2} + elif [ ! -z ${BOGO} ]; then + speed=${BOGO} + fi # Determine OS and kernel version - local os_name=$( uname -s 2>/dev/null ) - local os_release=$( uname -r 2>/dev/null ) + local os_name=$( uname -s 2>/dev/null ) + local os_release=$( uname -r 2>/dev/null ) - declare -a arr + declare -a arr - local dist_id="" + local dist_id="" # Determine OS distribution ID - if [ lsb_pres == "yes" ]; then - arr=( $( lsb_release -i 2>/dev/null ) ) - dist_id=${arr[2]} - else - local dist_id=$(cat /etc/centos-release | awk '{print $1}' 2>/dev/null) - fi - - local dist_rel="" - if [ lsb_pres == "yes" ]; then + if [ lsb_pres == "yes" ]; then + arr=( $( lsb_release -i 2>/dev/null ) ) + dist_id=${arr[2]} + else + local dist_id=$(cat /etc/centos-release | awk '{print $1}' 2>/dev/null) + fi + + local dist_rel="" + if [ lsb_pres == "yes" ]; then # Determine OS distribution release - arr=( $( cat /proc/version | awk '{print $3}' 2>/dev/null ) ) - local dist_rel=${arr[1]} - else - local dist_rel=$(cat /etc/centos-release | awk '{print $4}' 2>/dev/null) - fi + arr=( $( cat /proc/version | awk '{print $3}' 2>/dev/null ) ) + local dist_rel=${arr[1]} + else + local dist_rel=$(cat /etc/centos-release | awk '{print $4}' 2>/dev/null) + fi # Print generic header - echo "${TOOLNAME} -- ${tstamp} load average:${LDAVG[0]}, ${LDAVG[1]}, ${LDAVG[2]} runq:${LDAVG[3]} nproc:${LDAVG[4]}" - echo " host:${HOSTNAME} Distribution:${dist_id} ${dist_rel} ${os_name} ${os_release}" - echo " arch:${arch} processor:${processor} speed:${speed} MHz CPUs:${online_cpus}" + echo "${TOOLNAME} -- ${tstamp} load average:${LDAVG[0]}, ${LDAVG[1]}, ${LDAVG[2]} runq:${LDAVG[3]} nproc:${LDAVG[4]}" + echo " host:${HOSTNAME} Distribution:${dist_id} ${dist_rel} ${os_name} ${os_release}" + echo " arch:${arch} processor:${processor} speed:${speed} MHz CPUs:${online_cpus}" } @@ -394,85 +394,85 @@ function tools_header() { # tools_usage() - show generic tools tool usage function tools_usage() { - if [ ${OPT_USE_INTERVALS} -eq 1 ]; then - echo "usage: ${TOOLNAME} [-f] [-p ] [-i ] [-c ] [-h]" - else - echo "Usage: ${TOOLNAME} [-f] [-p ] [-c ] [-h]" - fi + if [ ${OPT_USE_INTERVALS} -eq 1 ]; then + echo "usage: ${TOOLNAME} [-f] [-p ] [-i ] [-c ] [-h]" + else + echo "Usage: ${TOOLNAME} [-f] [-p ] [-c ] [-h]" + fi } # tools_print_help() - print generic tool help function tools_print_help() { - tools_usage - echo - echo "Options:"; - echo " -f : collect forever : default: none" - echo " -p : overall collection period (minutes) : default: ${DEFAULT_PERIOD_MIN}" - if [ ${OPT_USE_INTERVALS} -eq 1 ]; then - echo " -i : sample interval (seconds) : default: ${DEFAULT_INTERVAL_SEC}" - fi - echo " -c : cpu list where tool runs (e.g., 0-1,8) : default: none" - echo - if [ ${OPT_USE_INTERVALS} -eq 1 ]; then - echo "Example: collect 5 minute period, sample every 30 seconds interval" - echo " ${TOOLNAME} -p 5 -i 30" - else - echo "Example: collect 5 minute period" - echo " ${TOOLNAME} -p 5" - fi + tools_usage + echo + echo "Options:"; + echo " -f : collect forever : default: none" + echo " -p : overall collection period (minutes) : default: ${DEFAULT_PERIOD_MIN}" + if [ ${OPT_USE_INTERVALS} -eq 1 ]; then + echo " -i : sample interval (seconds) : default: ${DEFAULT_INTERVAL_SEC}" + fi + echo " -c : cpu list where tool runs (e.g., 0-1,8) : default: none" + echo + if [ ${OPT_USE_INTERVALS} -eq 1 ]; then + echo "Example: collect 5 minute period, sample every 30 seconds interval" + echo " ${TOOLNAME} -p 5 -i 30" + else + echo "Example: collect 5 minute period" + echo " ${TOOLNAME} -p 5" + fi } # tools_parse_options() -- parse common options for tools scripts function tools_parse_options() { # check for no arguments, print usage - if [ $# -eq "0" ]; then - tools_usage - tools_cleanup 0 - exit 0 - fi + if [ $# -eq "0" ]; then + tools_usage + tools_cleanup 0 + exit 0 + fi # parse the input arguments - while getopts "fp:i:c:h" Option - do + while getopts "fp:i:c:h" Option + do case $Option in f) - OPT_FOREVER=1 - PERIOD_MIN=60 - ;; + OPT_FOREVER=1 + PERIOD_MIN=60 + ;; p) PERIOD_MIN=$OPTARG ;; i) - OPT_USE_INTERVALS=1 - INTERVAL_SEC=$OPTARG - ;; + OPT_USE_INTERVALS=1 + INTERVAL_SEC=$OPTARG + ;; c) CPULIST=$OPTARG ;; h) - tools_print_help - tools_cleanup 0 - exit 0 - ;; + tools_print_help + tools_cleanup 0 + exit 0 + ;; *) - tools_usage - tools_cleanup 0 - exit 0 - ;; + tools_usage + tools_cleanup 0 + exit 0 + ;; esac - done + done # validate input arguments - PERIOD_MAX=$[4*24*60] - INTERVAL_MAX=$[60*60] + PERIOD_MAX=$[4*24*60] + INTERVAL_MAX=$[60*60] - error=0 - if [[ ${PERIOD_MIN} -lt 1 || ${PERIOD_MIN} -gt ${PERIOD_MAX} ]]; then - echo "-p must be > 0 and <= ${PERIOD_MAX}." - error=1 - fi - if [[ ${INTERVAL_SEC} -lt 1 || ${INTERVAL_SEC} -gt ${INTERVAL_MAX} ]]; then - echo "-i must be > 0 and <= ${INTERVAL_MAX}." - error=1 - fi - if [ ${error} -eq 1 ]; then - tools_cleanup 0 - exit 1 - fi + error=0 + if [[ ${PERIOD_MIN} -lt 1 || ${PERIOD_MIN} -gt ${PERIOD_MAX} ]]; then + echo "-p must be > 0 and <= ${PERIOD_MAX}." + error=1 + fi + if [[ ${INTERVAL_SEC} -lt 1 || ${INTERVAL_SEC} -gt ${INTERVAL_MAX} ]]; then + echo "-i must be > 0 and <= ${INTERVAL_MAX}." + error=1 + fi + if [ ${error} -eq 1 ]; then + tools_cleanup 0 + exit 1 + fi } diff --git a/tools/engtools/hostdata-collectors/scripts/filestats.sh b/tools/engtools/hostdata-collectors/scripts/filestats.sh index 19d38a704..4f06b7562 100644 --- a/tools/engtools/hostdata-collectors/scripts/filestats.sh +++ b/tools/engtools/hostdata-collectors/scripts/filestats.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi PAGE_SIZE=$(getconf PAGE_SIZE) @@ -18,19 +18,19 @@ OPT_USE_INTERVALS=1 function print_files() { - print_separator + print_separator TOOL_HIRES_TIME - ${ECHO} "# ls -l /proc/*/fd" - sudo ls -l /proc/*/fd 2>/dev/null | awk \ - '$11 ~ /socket/ {a += 1} ; \ - $11 ~ /null/ {b += 1} ; \ - {c += 1} \ - END {\ - {printf "%-10s %-10s %-10s %-10s\n", "TOTAL", "FILES", "SOCKETS", "NULL PIPES"} \ - {printf "%-10s %-10s %-10s %-10s\n", c, c-(a+b) , a, b}}' + ${ECHO} "# ls -l /proc/*/fd" + sudo ls -l /proc/*/fd 2>/dev/null | awk \ + '$11 ~ /socket/ {a += 1} ; \ + $11 ~ /null/ {b += 1} ; \ + {c += 1} \ + END {\ + {printf "%-10s %-10s %-10s %-10s\n", "TOTAL", "FILES", "SOCKETS", "NULL PIPES"} \ + {printf "%-10s %-10s %-10s %-10s\n", c, c-(a+b) , a, b}}' - ${ECHO} + ${ECHO} ${ECHO} "# lsof" printf "%-7s %-7s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %s\n" "PID" "TOTAL" "FD" "U" "W" "R" "CWD" "RTD" "TXT" "MEM" "DEL" "TCP" "CMD" @@ -46,7 +46,7 @@ function print_files() {printf "%-7s %-7s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %-6s %s\n", \ pids[i]["PID"], \ pids[i]["TOTAL"],\ - ((pids[i]["u"]!="")? pids[i]["u"] : 0) + ((pids[i]["w"]!="")? pids[i]["w"] : 0 )+ ((pids[i]["r"]!="")? pids[i]["r"] : 0),\ + ((pids[i]["u"]!="")? pids[i]["u"] : 0) + ((pids[i]["w"]!="")? pids[i]["w"] : 0 )+ ((pids[i]["r"]!="")? pids[i]["r"] : 0),\ (pids[i]["u"]!="")? pids[i]["u"] : 0,\ (pids[i]["w"]!="")? pids[i]["w"] : 0,\ (pids[i]["r"]!="")? pids[i]["r"] : 0,\ @@ -56,13 +56,13 @@ function print_files() (pids[i]["mem"]!="")? pids[i]["mem"] : 0,\ (pids[i]["DEL"]!="")? pids[i]["DEL"] : 0,\ (pids[i]["TCP"]!="")? pids[i]["TCP"] : 0,\ - pids[i]["COMMAND"]} }}}' | sort -n -r -k3 + pids[i]["COMMAND"]} }}}' | sort -n -r -k3 - ${ECHO} + ${ECHO} - ${ECHO} "# lsof -nP +L1" - sudo lsof -nP +L1 - ${ECHO} + ${ECHO} "# lsof -nP +L1" + sudo lsof -nP +L1 + ${ECHO} } @@ -87,8 +87,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_files - sleep ${INTERVAL_SEC} + print_files + sleep ${INTERVAL_SEC} done print_files LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh b/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh index 6712e7976..f4c93ff67 100644 --- a/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh @@ -41,76 +41,76 @@ fi # Check for sufficient priviledges # [ JGAULD : possibly provide user = 'operator' option instead... ] if [ $UID -ne 0 ]; then - log_daemon_msg "Starting ${NAME} requires sudo/root access." - exit 1 + log_daemon_msg "Starting ${NAME} requires sudo/root access." + exit 1 fi case $1 in - start) + start) if [ -e ${PIDFILE} ]; then - pid=$(pidof -x ${NAME}) - if test "${pid}" != "" - then - echo_success "${NAME} already running" + pid=$(pidof -x ${NAME}) + if test "${pid}" != "" + then + echo_success "${NAME} already running" exit - fi + fi fi log_daemon_msg "Starting ${NAME}" if start-stop-daemon --start --background --quiet --oknodo --pidfile ${PIDFILE} \ - --exec ${DAEMON} -- ${DAEMON_ARGS} ; then - log_end_msg 0 + --exec ${DAEMON} -- ${DAEMON_ARGS} ; then + log_end_msg 0 else - log_end_msg 1 + log_end_msg 1 fi ;; - stop) + stop) if [ -e ${PIDFILE} ]; then - pids=$(pidof -x ${NAME}) - if [[ ! -z "${pids}" ]] - then - echo_success "Stopping ${NAME} [$pid]" - start-stop-daemon --stop --quiet --oknodo --pidfile ${PIDFILE} --retry=TERM/3/KILL/5 - # [ JGAULD: none of the following should be necessary ] - /usr/local/bin/cleanup-engtools.sh - else - echo_failure "${NAME} is not running" - fi + pids=$(pidof -x ${NAME}) + if [[ ! -z "${pids}" ]] + then + echo_success "Stopping ${NAME} [$pid]" + start-stop-daemon --stop --quiet --oknodo --pidfile ${PIDFILE} --retry=TERM/3/KILL/5 + # [ JGAULD: none of the following should be necessary ] + /usr/local/bin/cleanup-engtools.sh + else + echo_failure "${NAME} is not running" + fi else - echo_failure "${PIDFILE} does not exist" + echo_failure "${PIDFILE} does not exist" fi ;; - restart) + restart) $0 stop && sleep 2 && $0 start ;; - status) + status) if [ -e ${PIDFILE} ]; then - pid=$(pidof -x ${NAME}) - if test "${pid}" != "" - then - echo_success "${NAME} is running" - else - echo_success "${NAME} is not running" - fi + pid=$(pidof -x ${NAME}) + if test "${pid}" != "" + then + echo_success "${NAME} is running" + else + echo_success "${NAME} is not running" + fi else - echo_success "${NAME} is not running" + echo_success "${NAME} is not running" fi ;; - reload) + reload) if [ -e ${PIDFILE} ]; then - start-stop-daemon --stop --signal USR1 --quiet --pidfile ${PIDFILE} --name ${NAME} - echo_success "${NAME} reloaded successfully" + start-stop-daemon --stop --signal USR1 --quiet --pidfile ${PIDFILE} --name ${NAME} + echo_success "${NAME} reloaded successfully" else - echo_success "${PIDFILE} does not exist" + echo_success "${PIDFILE} does not exist" fi ;; - *) + *) echo "Usage: $0 {start|stop|restart|reload|status}" exit 2 ;; diff --git a/tools/engtools/hostdata-collectors/scripts/iostat.sh b/tools/engtools/hostdata-collectors/scripts/iostat.sh index 04be90c35..7ac65aceb 100644 --- a/tools/engtools/hostdata-collectors/scripts/iostat.sh +++ b/tools/engtools/hostdata-collectors/scripts/iostat.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -15,9 +15,9 @@ OPT_USE_INTERVALS=1 IOSTAT=$( which iostat 2>/dev/null ) if [ $? -ne 0 ]; then - print_separator - WARNLOG "iostat not available" - tools_cleanup 0 + print_separator + WARNLOG "iostat not available" + tools_cleanup 0 fi # MAIN Program: diff --git a/tools/engtools/hostdata-collectors/scripts/memstats.sh b/tools/engtools/hostdata-collectors/scripts/memstats.sh index 664b10b46..81e47777d 100644 --- a/tools/engtools/hostdata-collectors/scripts/memstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/memstats.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi PAGE_SIZE=$(getconf PAGE_SIZE) @@ -19,66 +19,66 @@ OPT_USE_INTERVALS=1 function print_memory() { # Configuration for netcmds - MEMINFO=/proc/meminfo - NODEINFO=/sys/devices/system/node/node?/meminfo - BUDDYINFO=/proc/buddyinfo - SLABINFO=/proc/slabinfo + MEMINFO=/proc/meminfo + NODEINFO=/sys/devices/system/node/node?/meminfo + BUDDYINFO=/proc/buddyinfo + SLABINFO=/proc/slabinfo - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME - ${ECHO} "# ${MEMINFO}" - ${CAT} ${MEMINFO} - ${ECHO} + ${ECHO} "# ${MEMINFO}" + ${CAT} ${MEMINFO} + ${ECHO} - ${ECHO} "# ${NODEINFO}" - ${CAT} ${NODEINFO} - ${ECHO} + ${ECHO} "# ${NODEINFO}" + ${CAT} ${NODEINFO} + ${ECHO} - ${ECHO} "# ${BUDDYINFO}" - ${CAT} ${BUDDYINFO} - ${ECHO} + ${ECHO} "# ${BUDDYINFO}" + ${CAT} ${BUDDYINFO} + ${ECHO} - ${ECHO} "# PSS" - cat /proc/*/smaps 2>/dev/null | \ + ${ECHO} "# PSS" + cat /proc/*/smaps 2>/dev/null | \ awk '/^Pss:/ {a += $2;} END {printf "%d MiB\n", a/1024.0;}' - ${ECHO} + ${ECHO} # use old slabinfo format (i.e. slub not enabled in kernel) - ${ECHO} "# ${SLABINFO}" - ${CAT} ${SLABINFO} | \ + ${ECHO} "# ${SLABINFO}" + ${CAT} ${SLABINFO} | \ awk -v page_size_B=${PAGE_SIZE} ' BEGIN {page_KiB = page_size_B/1024; TOT_KiB = 0;} (NF == 17) { - gsub(/[<>]/, ""); - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", + gsub(/[<>]/, ""); + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", $2, $3, $4, $5, $6, $7, $8, $10, $11, $12, $13, $15, $16, $17, "KiB"); } (NF == 16) { - num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; - KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; - TOT_KiB += KiB; - printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", + num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; + KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; + TOT_KiB += KiB; + printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", $1, $2, $3, $4, $5, $6, $7, $9, $10, $11, $12, $14, $15, $16, KiB); } END { - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", "TOTAL", "-", "-", "-", "-", "-", ":", "-", "-", "-", ":", "-", "-", "-", TOT_KiB); } ' 2>/dev/null - ${ECHO} + ${ECHO} - ${ECHO} "# disk usage: rootfs, tmpfs" - cmd='df -h -H -T --local -t rootfs -t tmpfs' - ${ECHO} "Disk space usage rootfs,tmpfs (SI):" - ${ECHO} "${cmd}" - ${cmd} - ${ECHO} + ${ECHO} "# disk usage: rootfs, tmpfs" + cmd='df -h -H -T --local -t rootfs -t tmpfs' + ${ECHO} "Disk space usage rootfs,tmpfs (SI):" + ${ECHO} "${cmd}" + ${cmd} + ${ECHO} - CMD='ps -e -o ppid,pid,nlwp,rss:10,vsz:10,cmd --sort=-rss' - ${ECHO} "# ${CMD}" - ${CMD} - ${ECHO} + CMD='ps -e -o ppid,pid,nlwp,rss:10,vsz:10,cmd --sort=-rss' + ${ECHO} "# ${CMD}" + ${CMD} + ${ECHO} } #------------------------------------------------------------------------------- @@ -101,8 +101,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_memory - sleep ${INTERVAL_SEC} + print_memory + sleep ${INTERVAL_SEC} done print_memory LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/netstats.sh b/tools/engtools/hostdata-collectors/scripts/netstats.sh index 4ed13b04e..cd306f743 100644 --- a/tools/engtools/hostdata-collectors/scripts/netstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/netstats.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -17,22 +17,22 @@ OPT_USE_INTERVALS=1 function print_netcmds() { # Configuration for netcmds - DEV=/proc/net/dev - NETSTAT=/proc/net/netstat + DEV=/proc/net/dev + NETSTAT=/proc/net/netstat - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME - for net in \ + for net in \ ${DEV} ${NETSTAT} - do - if [ -e "${net}" ] - then - ${ECHO} "# ${net}" - ${CAT} ${net} - ${ECHO} - fi - done + do + if [ -e "${net}" ] + then + ${ECHO} "# ${net}" + ${CAT} ${net} + ${ECHO} + fi + done } #------------------------------------------------------------------------------- @@ -55,8 +55,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_netcmds - sleep ${INTERVAL_SEC} + print_netcmds + sleep ${INTERVAL_SEC} done print_netcmds LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/postgres.sh b/tools/engtools/hostdata-collectors/scripts/postgres.sh index 9bcf8d1f3..7ee0a8577 100644 --- a/tools/engtools/hostdata-collectors/scripts/postgres.sh +++ b/tools/engtools/hostdata-collectors/scripts/postgres.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -16,32 +16,32 @@ OPT_USE_INTERVALS=1 # Print key networking device statistics function print_postgres() { - print_separator - TOOL_HIRES_TIME - + print_separator + TOOL_HIRES_TIME + # postgressql command: set user, disable pagination, and be quiet - PSQL="sudo -u postgres psql --pset pager=off -q" + PSQL="sudo -u postgres psql --pset pager=off -q" # List postgres databases - db_list=( $(${PSQL} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) - ${ECHO} "# postgres databases" - echo "db_list = ${db_list[@]}" - ${ECHO} + db_list=( $(${PSQL} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) + ${ECHO} "# postgres databases" + echo "db_list = ${db_list[@]}" + ${ECHO} # List sizes of all postgres databases (similar to "\l+") - ${ECHO} "# postgres database sizes" - ${PSQL} -c " + ${ECHO} "# postgres database sizes" + ${PSQL} -c " SELECT - pg_database.datname, - pg_database_size(pg_database.datname), - pg_size_pretty(pg_database_size(pg_database.datname)) + pg_database.datname, + pg_database_size(pg_database.datname), + pg_size_pretty(pg_database_size(pg_database.datname)) FROM pg_database ORDER BY pg_database_size DESC; " # For each database, list tables and their sizes (similar to "\dt+") - for db in "${db_list[@]}" - do + for db in "${db_list[@]}" + do ${ECHO} "# postgres database: ${db}" ${PSQL} -d ${db} -c " SELECT @@ -85,29 +85,29 @@ SELECT last_autoanalyze FROM pg_stat_user_tables; " - done + done # Specific table counts (This is very SLOW, look at "live tuples" instead) # Number of keystone tokens #${ECHO} "# keystone token count" # Number of postgres connections - ${ECHO} "# postgres database connections" - CONN=$(ps -C postgres -o cmd= | wc -l) - CONN_T=$(ps -C postgres -o cmd= | awk '/postgres: / {print $3}' | awk '{for(i=1;i<=NF;i++) a[$i]++} END {for(k in a) print k, a[k]}' | sort -k 2 -nr ) - ${ECHO} "connections total = ${CONN}" - ${ECHO} - ${ECHO} "connections breakdown:" - ${ECHO} "${CONN_T}" - ${ECHO} + ${ECHO} "# postgres database connections" + CONN=$(ps -C postgres -o cmd= | wc -l) + CONN_T=$(ps -C postgres -o cmd= | awk '/postgres: / {print $3}' | awk '{for(i=1;i<=NF;i++) a[$i]++} END {for(k in a) print k, a[k]}' | sort -k 2 -nr ) + ${ECHO} "connections total = ${CONN}" + ${ECHO} + ${ECHO} "connections breakdown:" + ${ECHO} "${CONN_T}" + ${ECHO} - ${ECHO} "connections breakdown (query):" - ${PSQL} -c "SELECT datname,state,count(*) from pg_stat_activity group by datname,state;" - ${ECHO} + ${ECHO} "connections breakdown (query):" + ${PSQL} -c "SELECT datname,state,count(*) from pg_stat_activity group by datname,state;" + ${ECHO} - ${ECHO} "connections idle age:" - ${PSQL} -c "SELECT datname,age(now(),state_change) from pg_stat_activity where state='idle';" - ${ECHO} + ${ECHO} "connections idle age:" + ${PSQL} -c "SELECT datname,age(now(),state_change) from pg_stat_activity where state='idle';" + ${ECHO} } #------------------------------------------------------------------------------- @@ -130,8 +130,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_postgres - sleep ${INTERVAL_SEC} + print_postgres + sleep ${INTERVAL_SEC} done print_postgres LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh b/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh index c588c16c5..64e0de9cb 100644 --- a/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh +++ b/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -17,8 +17,8 @@ MQOPT="-n rabbit@localhost" # Print key networking device statistics function print_rabbitmq() { - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME # IMPORTANT: # - Difficulty getting rabbitmqctl to work from init.d script; @@ -27,31 +27,31 @@ function print_rabbitmq() # - WORKAROUND: run command using 'sudo', even if you are 'root' # Dump various rabbitmq related stats - MQ_STATUS="rabbitmqctl ${MQOPT} status" - ${ECHO} "# ${MQ_STATUS}" - sudo ${MQ_STATUS} | grep -e '{memory' -A30 - ${ECHO} + MQ_STATUS="rabbitmqctl ${MQOPT} status" + ${ECHO} "# ${MQ_STATUS}" + sudo ${MQ_STATUS} | grep -e '{memory' -A30 + ${ECHO} # THe following is useful in diagnosing rabbit memory leaks # when end-users do not drain their queues (eg, due to RPC timeout issues, etc) - MQ_QUEUES="rabbitmqctl ${MQOPT} list_queues messages name pid messages_ready messages_unacknowledged memory consumers" - ${ECHO} "# ${MQ_QUEUES}" - sudo ${MQ_QUEUES} - ${ECHO} + MQ_QUEUES="rabbitmqctl ${MQOPT} list_queues messages name pid messages_ready messages_unacknowledged memory consumers" + ${ECHO} "# ${MQ_QUEUES}" + sudo ${MQ_QUEUES} + ${ECHO} - num_queues=$(sudo rabbitmqctl ${MQOPT} list_queues | wc -l); ((num_queues-=2)) - num_bindings=$(sudo rabbitmqctl ${MQOPT} list_bindings | wc -l); ((num_bindings-=2)) - num_exchanges=$(sudo rabbitmqctl ${MQOPT} list_exchanges | wc -l); ((num_exchanges-=2)) - num_connections=$(sudo rabbitmqctl ${MQOPT} list_connections | wc -l); ((num_connections-=2)) - num_channels=$(sudo rabbitmqctl ${MQOPT} list_channels | wc -l); ((num_channels-=2)) - arr=($(sudo rabbitmqctl ${MQOPT} list_queues messages consumers memory | \ - awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) - messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]} - printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" \ + num_queues=$(sudo rabbitmqctl ${MQOPT} list_queues | wc -l); ((num_queues-=2)) + num_bindings=$(sudo rabbitmqctl ${MQOPT} list_bindings | wc -l); ((num_bindings-=2)) + num_exchanges=$(sudo rabbitmqctl ${MQOPT} list_exchanges | wc -l); ((num_exchanges-=2)) + num_connections=$(sudo rabbitmqctl ${MQOPT} list_connections | wc -l); ((num_connections-=2)) + num_channels=$(sudo rabbitmqctl ${MQOPT} list_channels | wc -l); ((num_channels-=2)) + arr=($(sudo rabbitmqctl ${MQOPT} list_queues messages consumers memory | \ + awk '/^[0-9]/ {a+=$1; b+=$2; c+=$3} END {print a, b, c}')) + messages=${arr[0]}; consumers=${arr[1]}; memory=${arr[2]} + printf "%6s %8s %9s %11s %8s %8s %9s %10s\n" \ "queues" "bindings" "exchanges" "connections" "channels" "messages" "consumers" "memory" - printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" \ + printf "%6d %8d %9d %11d %8d %8d %9d %10d\n" \ $num_queues $num_bindings $num_exchanges $num_connections $num_channels $messages $consumers $memory - ${ECHO} + ${ECHO} } #------------------------------------------------------------------------------- @@ -74,8 +74,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_rabbitmq - sleep ${INTERVAL_SEC} + print_rabbitmq + sleep ${INTERVAL_SEC} done print_rabbitmq LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh b/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh index 3d972c477..e5ccd1ab2 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rbzip2-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -22,24 +22,24 @@ STORAGE=( $(system host-list | awk '(/storage/) {print $4;}') ) LOG "Remote bzip2 engtools data on all blades:" for blade in ${CONTROLLER[@]}; do - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "bzip2 on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo bzip2 /scratch/syseng_data/${blade}/* - else - WARNLOG "cannot ping: ${blade}" - fi + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "bzip2 on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo bzip2 /scratch/syseng_data/${blade}/* + else + WARNLOG "cannot ping: ${blade}" + fi done for blade in ${STORAGE[@]} ${COMPUTE[@]} ; do - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "bzip2 on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo bzip2 /tmp/syseng_data/${blade}/* - else - WARNLOG "cannot ping: ${blade}" - fi + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "bzip2 on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo bzip2 /tmp/syseng_data/${blade}/* + else + WARNLOG "cannot ping: ${blade}" + fi done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh b/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh index f3df76d55..21a47a4f3 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rstart-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -18,19 +18,19 @@ BLADES=( $(system host-list | awk '(/compute|controller|storage/) {print $4;}') LOG "Remote start engtools on all blades:" for blade in ${BLADES[@]}; do - if [ "${blade}" == "${HOSTNAME}" ]; then - LOG "start on $blade:" - sudo service collect-engtools.sh start - else - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "start on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo service collect-engtools.sh start + if [ "${blade}" == "${HOSTNAME}" ]; then + LOG "start on $blade:" + sudo service collect-engtools.sh start else - WARNLOG "cannot ping: ${blade}" + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "start on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo service collect-engtools.sh start + else + WARNLOG "cannot ping: ${blade}" + fi fi - fi done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh b/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh index 1251ea827..5637bfd22 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rstop-engtools.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -18,19 +18,19 @@ BLADES=( $(system host-list | awk '(/compute|controller|storage/) {print $4;}') LOG "Remote stop engtools on all blades:" for blade in ${BLADES[@]}; do - if [ "${blade}" == "${HOSTNAME}" ]; then - LOG "stop on $blade:" - sudo service collect-engtools.sh stop - else - ping -c1 ${blade} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "stop on $blade:" - ssh -q -t -o StrictHostKeyChecking=no \ - ${blade} sudo service collect-engtools.sh stop + if [ "${blade}" == "${HOSTNAME}" ]; then + LOG "stop on $blade:" + sudo service collect-engtools.sh stop else - WARNLOG "cannot ping: ${blade}" + ping -c1 ${blade} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "stop on $blade:" + ssh -q -t -o StrictHostKeyChecking=no \ + ${blade} sudo service collect-engtools.sh stop + else + WARNLOG "cannot ping: ${blade}" + fi fi - fi done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh b/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh index 6f82f47e7..2e6704293 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh if [ $UID -eq 0 ]; then - ERRLOG "Do not start $0 using sudo/root access." - exit 1 + ERRLOG "Do not start $0 using sudo/root access." + exit 1 fi # environment for system commands @@ -24,10 +24,10 @@ STORAGE=( $(system host-list | awk '(/storage/) {print $4;}') ) DEST=/opt/backups/syseng_data/ if [[ "${HOSTNAME}" =~ "controller-" ]]; then - LOG "rsync DEST=${DEST}" + LOG "rsync DEST=${DEST}" else - LOG "*ERROR* only run this on controller" - exit 1 + LOG "*ERROR* only run this on controller" + exit 1 fi sudo mkdir -p ${DEST} @@ -43,13 +43,13 @@ SRC=/scratch/syseng_data/ DEST=/opt/backups/syseng_data/ for HOST in ${CONTROLLER[@]} do - ping -c1 ${HOST} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" - sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} - else - WARNLOG "cannot ping: ${HOST}" - fi + ping -c1 ${HOST} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" + sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} + else + WARNLOG "cannot ping: ${HOST}" + fi done # computes & storage @@ -57,13 +57,13 @@ SRC=/tmp/syseng_data/ DEST=/opt/backups/syseng_data/ for HOST in ${STORAGE[@]} ${COMPUTE[@]} do - ping -c1 ${HOST} 1>/dev/null 2>/dev/null - if [ $? -eq 0 ]; then - LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" - sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} - else - WARNLOG "cannot ping: ${HOST}" - fi + ping -c1 ${HOST} 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" + sudo rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST} + else + WARNLOG "cannot ping: ${HOST}" + fi done LOG 'done' diff --git a/tools/engtools/hostdata-collectors/scripts/slab.sh b/tools/engtools/hostdata-collectors/scripts/slab.sh index 70e9c0596..f60e0c095 100644 --- a/tools/engtools/hostdata-collectors/scripts/slab.sh +++ b/tools/engtools/hostdata-collectors/scripts/slab.sh @@ -3,19 +3,19 @@ PAGE_SIZE=$(getconf PAGE_SIZE) cat /proc/slabinfo | awk -v page_size_B=${PAGE_SIZE} ' BEGIN {page_KiB = page_size_B/1024; TOT_KiB = 0;} (NF == 17) { - gsub(/[<>]/, ""); - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", + gsub(/[<>]/, ""); + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8s\n", $2, $3, $4, $5, $6, $7, $8, $10, $11, $12, $13, $15, $16, $17, "KiB"); } (NF == 16) { - num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; - KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; - TOT_KiB += KiB; - printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", + num_objs=$3; obj_per_slab=$5; pages_per_slab=$6; + KiB = (obj_per_slab > 0) ? page_KiB*num_objs/obj_per_slab*pages_per_slab : 0; + TOT_KiB += KiB; + printf("%-22s %11d %8d %8d %10d %12d %1s %5d %10d %12d %1s %12d %9d %11d %8d\n", $1, $2, $3, $4, $5, $6, $7, $9, $10, $11, $12, $14, $15, $16, KiB); } END { - printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", + printf("%-22s %11s %8s %8s %10s %12s %1s %5s %10s %12s %1s %12s %9s %11s %8d\n", "TOTAL", "-", "-", "-", "-", "-", ":", "-", "-", "-", ":", "-", "-", "-", TOT_KiB); } ' 2>/dev/null diff --git a/tools/engtools/hostdata-collectors/scripts/ticker.sh b/tools/engtools/hostdata-collectors/scripts/ticker.sh index 570cd526d..0bfe8ad68 100644 --- a/tools/engtools/hostdata-collectors/scripts/ticker.sh +++ b/tools/engtools/hostdata-collectors/scripts/ticker.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -31,15 +31,15 @@ LOG_NOCR "collecting " t=0 for ((rep=1; rep <= REPEATS ; rep++)) do - ((t++)) - sleep ${INTERVAL_SEC} - if [ ${t} -ge ${REP_LOG} ]; then - t=0 - echo "." - LOG_NOCR "collecting " - else - echo -n "." - fi + ((t++)) + sleep ${INTERVAL_SEC} + if [ ${t} -ge ${REP_LOG} ]; then + t=0 + echo "." + LOG_NOCR "collecting " + else + echo -n "." + fi done echo "." diff --git a/tools/engtools/hostdata-collectors/scripts/top.sh b/tools/engtools/hostdata-collectors/scripts/top.sh index 45dff33d7..9a5ebc0cd 100644 --- a/tools/engtools/hostdata-collectors/scripts/top.sh +++ b/tools/engtools/hostdata-collectors/scripts/top.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval diff --git a/tools/engtools/hostdata-collectors/scripts/vswitch.sh b/tools/engtools/hostdata-collectors/scripts/vswitch.sh index dae8bf76a..1e6a5a338 100644 --- a/tools/engtools/hostdata-collectors/scripts/vswitch.sh +++ b/tools/engtools/hostdata-collectors/scripts/vswitch.sh @@ -6,8 +6,8 @@ TOOLBIN=$(dirname $0) . ${TOOLBIN}/engtools_util.sh tools_init if [ $? -ne 0 ]; then - echo "FATAL, tools_init - could not setup environment" - exit $? + echo "FATAL, tools_init - could not setup environment" + exit $? fi # Enable use of INTERVAL_SEC sample interval @@ -16,25 +16,25 @@ OPT_USE_INTERVALS=1 # Print key networking device statistics function print_vswitch() { - print_separator - TOOL_HIRES_TIME + print_separator + TOOL_HIRES_TIME - cmd='vshell engine-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell engine-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell port-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell port-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell network-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell network-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell interface-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} - cmd='vshell interface-stats-list' - ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell engine-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell engine-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell port-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell port-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell network-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell network-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell interface-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} + cmd='vshell interface-stats-list' + ${ECHO} "# ${cmd}" ; ${cmd} ; ${ECHO} } #------------------------------------------------------------------------------- @@ -57,8 +57,8 @@ tools_header for ((rep=1; rep <= REPEATS ; rep++)) do - print_vswitch - sleep ${INTERVAL_SEC} + print_vswitch + sleep ${INTERVAL_SEC} done print_vswitch LOG "done" diff --git a/tools/engtools/parsers/common/cleanup-uncompressed.sh b/tools/engtools/parsers/common/cleanup-uncompressed.sh index 3bd1fa21f..26824b269 100755 --- a/tools/engtools/parsers/common/cleanup-uncompressed.sh +++ b/tools/engtools/parsers/common/cleanup-uncompressed.sh @@ -5,15 +5,15 @@ #SPDX-License-Identifier: Apache-2.0 # # This script removes uncompressed file. It can save a huge amount of disk space -# on the analysis server. Run this script after the very last time the data is parsed -# and BEFORE running parse-daily.sh script. -# If it is run after each intermediary parse, the download-data.sh script will download the +# on the analysis server. Run this script after the very last time the data is parsed +# and BEFORE running parse-daily.sh script. +# If it is run after each intermediary parse, the download-data.sh script will download the # uncompressed files again. if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf diff --git a/tools/engtools/parsers/common/download-data.sh b/tools/engtools/parsers/common/download-data.sh index 0c5726dd1..7ac8c3d21 100755 --- a/tools/engtools/parsers/common/download-data.sh +++ b/tools/engtools/parsers/common/download-data.sh @@ -14,9 +14,9 @@ # This script is to be run after running download-computes.sh on one of the controllers. if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf @@ -38,6 +38,6 @@ for HOST in ${ALL_HOSTS}; do bzip2 ${HOST}* cd ${CURDIR} else - echo "${HOST} not found" + echo "${HOST} not found" fi done diff --git a/tools/engtools/parsers/common/parse-all.sh b/tools/engtools/parsers/common/parse-all.sh index 99d1e4eb5..42f732dcc 100755 --- a/tools/engtools/parsers/common/parse-all.sh +++ b/tools/engtools/parsers/common/parse-all.sh @@ -8,7 +8,7 @@ # This script is used to parse all stats data. It is designed to be called by either # parse-controllers.sh or parse-computes.sh and not used as a standalone script. # If the input node is a controller, it will parse controller specific postgres & -# and rabbitmq stats first. If the input node is a compute, it will pars the compute +# and rabbitmq stats first. If the input node is a compute, it will pars the compute # specific vswitch stats first. # # The following parsing steps are common to all hosts and are executed in the specified order: @@ -22,12 +22,12 @@ # - Parse filestats (summary) # - Parse process level schedtop (optional step, configured in lab.conf) # - Generate tarball - + if [[ $# != 1 ]]; then - echo "ERROR: This script is meant to be called by either parse-controllers.sh or parse-computes.sh script." - echo "To run it separately, copy the script to the host directory that contains *.bz2 files." - echo "It takes a single argument - the name of the host directory (e.g. ./parse-all.sh controller-0)." - exit 1 + echo "ERROR: This script is meant to be called by either parse-controllers.sh or parse-computes.sh script." + echo "To run it separately, copy the script to the host directory that contains *.bz2 files." + echo "It takes a single argument - the name of the host directory (e.g. ./parse-all.sh controller-0)." + exit 1 fi source ../lab.conf @@ -49,7 +49,7 @@ function sedit() # Remove any trailing comma sed -i "s/,$//" ${FILETOSED} } - + function get_filename_from_mountname() { local name=$1 @@ -153,7 +153,7 @@ function parse_occtop_data() echo "${header}" > occtop-${NODE}-detailed.csv cat tmp2.txt >> occtop-${NODE}-detailed.csv - # Generate simple CSV file which is used to generate host CPU occupancy chart. Platform cores are + # Generate simple CSV file which is used to generate host CPU occupancy chart. Platform cores are # defined in the host.conf. The simple CSV contains only the Date/Time and Total platform CPU occupancy e.g. # Date/Time,Total # 2016-11-22 00:29:16.523,94.9 @@ -187,7 +187,7 @@ function parse_memtop_data() bzcat *memtop.bz2 > memtop-${NODE}-${DATESTAMP}.txt cp memtop-${NODE}-${DATESTAMP}.txt tmp.txt sedit tmp.txt - + # After dumping all memtop bz2 output into one text file and in-place sed, grab only relevant data # for CSV output. Generate both detailed and simple CSV files. Simple output will be used to generate # chart. @@ -222,7 +222,7 @@ function parse_netstats_data() done < tmp.txt done rm tmp.txt - fi + fi } function parse_iostats_data() @@ -238,28 +238,28 @@ function parse_iostats_data() bzcat *iostat.bz2 | grep -E "/2015|/2016|/2017|${DEVICE}" | awk '{print $1","$2","$3","$4","$5","$6","$7","$8","$9","$10","$11","$12","$13","$14}' > tmp.txt while IFS= read -r current do - if test "${current#*Linux}" != "$current" - then - # Skip the line that contains the word "Linux" - continue - else - if test "${current#*$DEVICE}" == "$current" - then - # It's a date entry, look ahead - read -r next - if test "${next#*$DEVICE}" != "${next}" - then - # This next line contains the device stats - # Combine date and time fields - current="${current//2016,/2016 }" - current="${current//2017,/2017 }" - # Combine time and AM/PM fields - current="${current//,AM/ AM}" - current="${current//,PM/ PM}" - # Write both lines to intermediate file - echo "${current}" >> tmp2.txt - echo "${next}" >> tmp2.txt - fi + if test "${current#*Linux}" != "$current" + then + # Skip the line that contains the word "Linux" + continue + else + if test "${current#*$DEVICE}" == "$current" + then + # It's a date entry, look ahead + read -r next + if test "${next#*$DEVICE}" != "${next}" + then + # This next line contains the device stats + # Combine date and time fields + current="${current//2016,/2016 }" + current="${current//2017,/2017 }" + # Combine time and AM/PM fields + current="${current//,AM/ AM}" + current="${current//,PM/ PM}" + # Write both lines to intermediate file + echo "${current}" >> tmp2.txt + echo "${next}" >> tmp2.txt + fi fi fi done < tmp.txt @@ -272,7 +272,7 @@ function parse_iostats_data() cut -d, -f2-11 --complement tmp2.txt > tmp.txt # Write final content to output csv cat tmp.txt >> iostat-${NODE}-${DEVICE}.csv - rm tmp.txt tmp2.txt + rm tmp.txt tmp2.txt done fi } @@ -317,9 +317,9 @@ parse_occtop_data # Parsing memtop data parse_memtop_data -# Parsing memstats data to generate the high level report. The most important piece of info is the list of +# Parsing memstats data to generate the high level report. The most important piece of info is the list of # hi-runners at the end of the file. If there is a leak, run parse-daily.sh script to generate the time -# series data for the offending processes only. Use process name, not PID as most Titanium Cloud processes have +# series data for the offending processes only. Use process name, not PID as most Titanium Cloud processes have # workers. LOG "Parsing memstats summary for ${NODE}" ../parse_memstats --report *memstats.bz2 > memstats-summary-${NODE}-${DATESTAMP}.txt @@ -331,7 +331,7 @@ rm pid-*.csv parse_netstats_data # Parsing schedtop data to generate the high level report. Leave the process level schedtop parsing till -# the end as it is a long running task. +# the end as it is a long running task. LOG "Parsing schedtop summary for ${NODE}" FILES=$(ls *schedtop.bz2) ../parse_schedtop ${FILES} > schedtop-summary-${NODE}-${DATESTAMP}.txt @@ -342,17 +342,17 @@ parse_iostats_data # Parsing diskstats data parse_diskstats_data -# Parsing filestats data to generate the high level report. If there is a file descriptor leak, run parse-daily.sh -# script to generate the time series data for the offending processes only. Use process name, not PID as most +# Parsing filestats data to generate the high level report. If there is a file descriptor leak, run parse-daily.sh +# script to generate the time series data for the offending processes only. Use process name, not PID as most # Titanium Cloud processes have workers. LOG "Parsing filestats summary for ${NODE}" -../parse_filestats --all *filestats.bz2 > filestats-summary-${NODE}-${DATESTAMP}.txt +../parse_filestats --all *filestats.bz2 > filestats-summary-${NODE}-${DATESTAMP}.txt # Parsing process level schedtop data. This is a long running task. To skip this step or generate data for # only specific processes, update the lab.conf and host.conf files. [[ ${GENERATE_PROCESS_SCHEDTOP} == Y ]] && parse_process_schedtop_data || WARNLOG "Parsing process level schedtop is skipped." -# Done parsing for this host. If it's a controller host, check if the parsing of postgres connection stats which is run in +# Done parsing for this host. If it's a controller host, check if the parsing of postgres connection stats which is run in # parallel is done before creating a tar file. if test "${NODE#*"controller"}" != "${NODE}"; then # If postgres-conns.csv file has not been created which is highly unlikely, wait a couple of minutes diff --git a/tools/engtools/parsers/common/parse-controllers.sh b/tools/engtools/parsers/common/parse-controllers.sh index 61a77c585..da52b07ea 100755 --- a/tools/engtools/parsers/common/parse-controllers.sh +++ b/tools/engtools/parsers/common/parse-controllers.sh @@ -14,9 +14,9 @@ PARSERDIR=$(dirname $0) . ${PARSERDIR}/parse-util.sh if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf @@ -36,12 +36,12 @@ for HOST in ${CONTROLLER_LIST}; do sleep 120 cd .. else - ERRLOG "${HOST} does not exist. Parsing skipped." + ERRLOG "${HOST} does not exist. Parsing skipped." fi done # Parsing postgres connection stats is a time consuming step, run it in parallel with parse-all -# script. +# script. for HOST in ${CONTROLLER_LIST}; do if [ -d ${HOST} ]; then LOG "Parsing postgres connection stats data for controller host ${HOST}" diff --git a/tools/engtools/parsers/common/parse-daily.sh b/tools/engtools/parsers/common/parse-daily.sh index 7298441c3..971c5acdd 100755 --- a/tools/engtools/parsers/common/parse-daily.sh +++ b/tools/engtools/parsers/common/parse-daily.sh @@ -102,8 +102,8 @@ elif [[ $# -eq 3 ]]; then cd $1 parse_daily_stats $2 $3 else - echo "ERROR: Specified host $1 does not exist." - exit 1 + echo "ERROR: Specified host $1 does not exist." + exit 1 fi else echo "Specified parser $2 is not a valid parser." diff --git a/tools/engtools/parsers/common/parse-postgres.sh b/tools/engtools/parsers/common/parse-postgres.sh index 2e15e12ca..0d1a68d36 100755 --- a/tools/engtools/parsers/common/parse-postgres.sh +++ b/tools/engtools/parsers/common/parse-postgres.sh @@ -63,7 +63,7 @@ function generate_grep_str() { local grepstr="time:" for DB in ${DATABASE_LIST}; do - grepstr="${grepstr}|${DB}" + grepstr="${grepstr}|${DB}" done grepstr="${grepstr}|breakdown|connections total|rows" echo $grepstr diff --git a/tools/engtools/parsers/common/parse-rabbitmq.sh b/tools/engtools/parsers/common/parse-rabbitmq.sh index a5f19473d..01df49a8e 100755 --- a/tools/engtools/parsers/common/parse-rabbitmq.sh +++ b/tools/engtools/parsers/common/parse-rabbitmq.sh @@ -8,7 +8,7 @@ OUTFILE=$1 FILES=$(ls *rabbitmq.bz2 | sort) -[ -e tmp.txt ] && rm tmp.txt +[ -e tmp.txt ] && rm tmp.txt echo "Time/Date,Memory Total,Connection Readers,Connection Writers,Connection Channels,Connection Other,Queue Procs,Queue Slave Procs,Plugins,Other Proc,Mnesia,Mgmt DB,Msg Index,Other ETS,Binary,Code,Atom,Other System,VM Memory High Watermark,VM Memory Limit,Disk Free Limit,Disk Free,Descriptor Limit,Descriptors Used,Sockets Limit,Sockets Used,Processes Limit,Processes Used,Run Queue,Uptime" >${OUTFILE} diff --git a/tools/engtools/parsers/common/parse-schedtop.sh b/tools/engtools/parsers/common/parse-schedtop.sh index 90cd88346..cba3fcbba 100755 --- a/tools/engtools/parsers/common/parse-schedtop.sh +++ b/tools/engtools/parsers/common/parse-schedtop.sh @@ -48,7 +48,7 @@ function parse_schedtop_data() if [[ $# -eq 0 ]]; then # Parsing detail schedtop stats for all services configured in host.conf for all hosts would - # take a very long time and is often unnecessary. Until the performance issue with parse_schedtop + # take a very long time and is often unnecessary. Until the performance issue with parse_schedtop # is addressed, this is not supported. print_usage else diff --git a/tools/engtools/parsers/common/parse-util.sh b/tools/engtools/parsers/common/parse-util.sh index fa0c8f6d4..c469d1502 100755 --- a/tools/engtools/parsers/common/parse-util.sh +++ b/tools/engtools/parsers/common/parse-util.sh @@ -10,17 +10,17 @@ LOGFILE="parserlog.txt" function LOG () { - local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) - echo -e "${tstamp_H} $0($$): $@" >> ${LOGFILE} + local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) + echo -e "${tstamp_H} $0($$): $@" >> ${LOGFILE} } function ERRLOG () { - LOG "ERROR: $@" + LOG "ERROR: $@" } function WARNLOG () { - LOG "WARN: $@" + LOG "WARN: $@" } diff --git a/tools/engtools/parsers/large_office/download-computes.sh b/tools/engtools/parsers/large_office/download-computes.sh index 9e4806c01..fbceb79e3 100755 --- a/tools/engtools/parsers/large_office/download-computes.sh +++ b/tools/engtools/parsers/large_office/download-computes.sh @@ -6,14 +6,14 @@ # # Create /opt/backups/tmp/syseng-data directory on controller 0, change mode of this # directory to 777 and place this script and the lab.conf files there. It is recommended -# to set up password-less login from the controller to all storage and compute hosts +# to set up password-less login from the controller to all storage and compute hosts # before running the script. # if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf diff --git a/tools/engtools/parsers/large_office/parse-computes.sh b/tools/engtools/parsers/large_office/parse-computes.sh index 6addeaff4..60206c870 100755 --- a/tools/engtools/parsers/large_office/parse-computes.sh +++ b/tools/engtools/parsers/large_office/parse-computes.sh @@ -7,22 +7,22 @@ # This script is used to parse stats data for storage and compute hosts. It is not # relevant for CPE. For large office, it is called by parse-everything.sh script. -# File lab.conf must exist for the script to run. The STORAGE_LIST and COMPUTE_LIST -# config parameters can be set to suit the parsing needs. +# File lab.conf must exist for the script to run. The STORAGE_LIST and COMPUTE_LIST +# config parameters can be set to suit the parsing needs. PARSERDIR=$(dirname $0) . ${PARSERDIR}/parse-util.sh if [ ! -f lab.conf ]; then - echo "Lab configuration file is missing." - echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." - exit 1 + echo "Lab configuration file is missing." + echo "See http://wiki.wrs.com/PBUeng/TitaniumServerSysengToolsAndDataAnalysis for more info." + exit 1 fi source ./lab.conf if [ -z "${STORAGE_LIST}" ]; then - # This script could be invoked from another script or run separately so write to both + # This script could be invoked from another script or run separately so write to both # console and log file. echo "STORAGE_LIST is not set in lab.conf file. Skipping stats parsing for all storage." WARNLOG "STORAGE_LIST is not set in lab.conf file. Skipping stats parsing for all storage." @@ -35,7 +35,7 @@ else ../parse-all.sh ${HOST} > /dev/null 2>&1 & cd .. else - ERRLOG "${HOST} does not exist. Parsing skipped." + ERRLOG "${HOST} does not exist. Parsing skipped." fi done fi @@ -45,7 +45,7 @@ if [ -z "${COMPUTE_LIST}" ]; then WARNLOG "COMPUTE_LIST is not set in lab.conf file. Skipping stats parsing for all computes." exit 1 else - # If there is a large number of computes, they need to be parsed one batch at a time, + # If there is a large number of computes, they need to be parsed one batch at a time, # otherwise, the analysis server will be brought down to a crawl. Set the number of # computes to process in parallel as batches of 25 if it's not set in lab.conf BATCH_SIZE=${BATCH_SIZE:-25} diff --git a/utilities/build-info/build-info-1.0/collect.sh b/utilities/build-info/build-info-1.0/collect.sh index 8d06f6a62..48abc23d0 100755 --- a/utilities/build-info/build-info-1.0/collect.sh +++ b/utilities/build-info/build-info-1.0/collect.sh @@ -24,7 +24,7 @@ if [ ! -e $jenkinsBuildFile ]; then fi if [ -e $releaseInfoFile ]; then - source $releaseInfoFile + source $releaseInfoFile fi if [ "${REPO}x" == "x" ]; then @@ -35,12 +35,12 @@ if [ -e $jenkinsBuildFile ]; then cp $jenkinsBuildFile $destFile source $jenkinsBuildFile else - # PLATFORM_RELEASE should be set in release-info.inc - if [ "x${PLATFORM_RELEASE}" == "x" ]; then - SW_VERSION="Unknown" - else - SW_VERSION="${PLATFORM_RELEASE}" - fi + # PLATFORM_RELEASE should be set in release-info.inc + if [ "x${PLATFORM_RELEASE}" == "x" ]; then + SW_VERSION="Unknown" + else + SW_VERSION="${PLATFORM_RELEASE}" + fi BUILD_TARGET="Unknown" BUILD_TYPE="Informal" @@ -89,8 +89,8 @@ fi echo "#ifndef _BUILD_INFO_H_" > $destH echo "#define _BUILD_INFO_H_" >> $destH echo "" >> $destH -echo "#define RELEASE_NAME \"$RELEASE_NAME\"" >> $destH -echo "#define SW_VERSION \"$SW_VERSION\"" >> $destH +echo "#define RELEASE_NAME \"$RELEASE_NAME\"" >> $destH +echo "#define SW_VERSION \"$SW_VERSION\"" >> $destH echo "" >> $destH echo "#define BUILD_TARGET \"$BUILD_TARGET\"" >> $destH echo "#define BUILD_TYPE \"$BUILD_TYPE\"" >> $destH diff --git a/utilities/platform-util/scripts/cgcs_tc_setup.sh b/utilities/platform-util/scripts/cgcs_tc_setup.sh index 0630fac68..9507ae978 100755 --- a/utilities/platform-util/scripts/cgcs_tc_setup.sh +++ b/utilities/platform-util/scripts/cgcs_tc_setup.sh @@ -32,7 +32,7 @@ fi function test_valid_speed { # After the link is enabled but before the autonegotiation is complete - # the link speed may be read as either -1 or as 4294967295 (which is + # the link speed may be read as either -1 or as 4294967295 (which is # uint(-1) in twos-complement) depending on the kernel. Neither one is valid. if (( $1 > 0 )) && (( $1 != 4294967295 )) then @@ -62,16 +62,16 @@ function is_consolidated { if ! infra_exists then - return 1 + return 1 else # determine whether the management interface is a parent of the # infrastructure interface based on name. # eg. this matches enp0s8 to enp0s8.10 but not enp0s88 - if [[ $infrastructure_interface =~ $management_interface[\.][0-9]+$ ]] - then - return 0 - fi - return 1 + if [[ $infrastructure_interface =~ $management_interface[\.][0-9]+$ ]] + then + return 0 + fi + return 1 fi } diff --git a/utilities/platform-util/scripts/remotelogging_tc_setup.sh b/utilities/platform-util/scripts/remotelogging_tc_setup.sh index 3f8f8e459..4f44fa240 100755 --- a/utilities/platform-util/scripts/remotelogging_tc_setup.sh +++ b/utilities/platform-util/scripts/remotelogging_tc_setup.sh @@ -34,7 +34,7 @@ function log function test_valid_speed { # After the link is enabled but before the autonegotiation is complete - # the link speed may be read as either -1 or as 4294967295 (which is + # the link speed may be read as either -1 or as 4294967295 (which is # uint(-1) in twos-complement) depending on the kernel. Neither one is valid. if (( $1 > 0 )) && (( $1 != 4294967295 )) then diff --git a/virt/qemu/centos/files/build_configure.sh b/virt/qemu/centos/files/build_configure.sh index 31d78a19c..f9ba586f4 100755 --- a/virt/qemu/centos/files/build_configure.sh +++ b/virt/qemu/centos/files/build_configure.sh @@ -41,11 +41,11 @@ shift if [ "$have_rbd" == "enable" ]; then - rbd_driver=rbd, + rbd_driver=rbd, fi if [ "$have_gluster" == "enable" ]; then - gluster_driver=gluster, + gluster_driver=gluster, fi ./configure \ From 7cd4bf4c76c53ebaf0deba14ffde5ef5065d52ca Mon Sep 17 00:00:00 2001 From: zhangyangyang Date: Sat, 1 Sep 2018 17:34:05 +0800 Subject: [PATCH 2/9] Python 3 compatibility: fix xrange/range issues xrange is not defined in python3. Rename xrange() to range(). Story: 2002909 Task: 24567 Change-Id: I1cf9b77339fee79fd4750ca873a952a39974c23c Signed-off-by: zhangyangyang --- .../vm-topology/vm_topology/exec/vm_topology.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/vm-topology/vm-topology/vm_topology/exec/vm_topology.py b/tools/vm-topology/vm-topology/vm_topology/exec/vm_topology.py index 3dbb32a7b..011900bce 100644 --- a/tools/vm-topology/vm-topology/vm_topology/exec/vm_topology.py +++ b/tools/vm-topology/vm-topology/vm_topology/exec/vm_topology.py @@ -444,7 +444,7 @@ def _mask_to_cpulist(mask=0): # Assume max number of cpus for now... max_cpus = 128 - for cpu in xrange(max_cpus): + for cpu in range(max_cpus): if ((1 << cpu) & mask): cpulist.append(cpu) return cpulist @@ -492,9 +492,9 @@ def range_to_list(csv_range=None): """ if not csv_range: return [] - xranges = [(lambda L: xrange(L[0], L[-1] + 1))(map(int, r.split('-'))) + ranges = [(lambda L: range(L[0], L[-1] + 1))(map(int, r.split('-'))) for r in csv_range.split(',')] - return [y for x in xranges for y in x] + return [y for x in ranges for y in x] class TimeoutError(Exception): @@ -613,7 +613,7 @@ def do_libvirt_domain_info((host)): cpulist_d = {} cpuset_total = 0 up_total = 0 - for vcpu in xrange(d_nrVirtCpu): + for vcpu in range(d_nrVirtCpu): cpuset_b = d_vcpus[1][vcpu] cpuset = 0 for cpu, up in enumerate(cpuset_b): From a98bf512c4907e05c62188bdff8fbf6510f4656c Mon Sep 17 00:00:00 2001 From: Bin Qian Date: Tue, 4 Sep 2018 13:44:27 -0400 Subject: [PATCH 3/9] Remove OCF script code to monitor I/F status Remove OCF script code to monitor I/F status and fail the service when the I/F is down. On the other change, the monitor work is done by SM, to perform swact based on healthy state of both controllers. Depends: Ifa7453136f34768b99e2bcd741d1065e69ef452e Story: 2003577 Task: 24900 Change-Id: Ie3da54940054ef5c1880dae718272be1663026d8 Signed-off-by: Bin Qian --- .../centos/meta_patches/PATCH_ORDER | 1 + ...addr2-avoid-failing-svc-if-down-meta.patch | 32 ++++++++++ .../ipaddr2-avoid-failing-svc-if-down.patch | 61 +++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 base/cluster-resource-agents/centos/meta_patches/ipaddr2-avoid-failing-svc-if-down-meta.patch create mode 100644 base/cluster-resource-agents/centos/patches/ipaddr2-avoid-failing-svc-if-down.patch diff --git a/base/cluster-resource-agents/centos/meta_patches/PATCH_ORDER b/base/cluster-resource-agents/centos/meta_patches/PATCH_ORDER index 7750a32fa..22608823f 100644 --- a/base/cluster-resource-agents/centos/meta_patches/PATCH_ORDER +++ b/base/cluster-resource-agents/centos/meta_patches/PATCH_ORDER @@ -11,3 +11,4 @@ spec-add-ipaddr2-ignore-lo-state.patch Modify-error-code-of-bz1454699-fix-to-prevent-inactive-controller-reboot-loop.patch Disable-creation-of-the-debug-package.patch metapatch-for-arp_bg.patch +ipaddr2-avoid-failing-svc-if-down-meta.patch diff --git a/base/cluster-resource-agents/centos/meta_patches/ipaddr2-avoid-failing-svc-if-down-meta.patch b/base/cluster-resource-agents/centos/meta_patches/ipaddr2-avoid-failing-svc-if-down-meta.patch new file mode 100644 index 000000000..5f1c0b88b --- /dev/null +++ b/base/cluster-resource-agents/centos/meta_patches/ipaddr2-avoid-failing-svc-if-down-meta.patch @@ -0,0 +1,32 @@ +From a8fc00f7d84327284643f70638da2201327bdd10 Mon Sep 17 00:00:00 2001 +From: Bin Qian +Date: Wed, 29 Aug 2018 11:22:32 -0400 +Subject: [PATCH 1/1] ipaddr2 avoid failing when svc i/f down + +--- + SPECS/resource-agents.spec | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/SPECS/resource-agents.spec b/SPECS/resource-agents.spec +index aa06c49..0eca09f 100644 +--- a/SPECS/resource-agents.spec ++++ b/SPECS/resource-agents.spec +@@ -257,6 +257,7 @@ Patch1118: ipaddr2_if_down.patch + Patch1119: ipaddr2_ignore_lo_if_state.patch + Patch1120: Modify-error-code-of-bz1454699-fix-to-prevent-inactive-controller-reboot-loop.patch + Patch1121: Re-enable-background-execution-of-arp-commands.patch ++Patch1122: ipaddr2-avoid-failing-svc-if-down.patch + + Obsoletes: heartbeat-resources <= %{version} + Provides: heartbeat-resources = %{version} +@@ -568,6 +569,7 @@ exit 1 + %patch1119 -p1 + %patch1120 -p1 + %patch1121 -p1 ++%patch1122 -p1 + + %build + if [ ! -f configure ]; then +-- +1.8.3.1 + diff --git a/base/cluster-resource-agents/centos/patches/ipaddr2-avoid-failing-svc-if-down.patch b/base/cluster-resource-agents/centos/patches/ipaddr2-avoid-failing-svc-if-down.patch new file mode 100644 index 000000000..fcc75030f --- /dev/null +++ b/base/cluster-resource-agents/centos/patches/ipaddr2-avoid-failing-svc-if-down.patch @@ -0,0 +1,61 @@ +From c3448b1536d50291dc5ca49dce5957c39403cc82 Mon Sep 17 00:00:00 2001 +From: Bin Qian +Date: Wed, 29 Aug 2018 11:00:22 -0400 +Subject: [PATCH 1/1] avoid failing service when I/F is down + +--- + heartbeat/IPaddr2 | 24 ++++++------------------ + 1 file changed, 6 insertions(+), 18 deletions(-) + +diff --git a/heartbeat/IPaddr2 b/heartbeat/IPaddr2 +index 86009b9..2da5c5e 100755 +--- a/heartbeat/IPaddr2 ++++ b/heartbeat/IPaddr2 +@@ -885,12 +885,8 @@ ip_start() { + then + exit $OCF_SUCCESS + else +- if [ "$OCF_RESKEY_dc" = "yes" ]; then +- ocf_log info "NIC $NIC is DOWN..." +- exit $OCF_SUCCESS +- else +- exit $OCF_ERR_GENERIC +- fi ++ ocf_log info "NIC $NIC is DOWN..." ++ exit $OCF_SUCCESS + fi + fi + +@@ -954,12 +950,8 @@ ip_start() { + then + exit $OCF_SUCCESS + else +- if [ "$OCF_RESKEY_dc" = "yes" ]; then +- ocf_log info "NIC $NIC is DOWN" +- exit $OCF_SUCCESS +- else +- exit $OCF_ERR_GENERIC +- fi ++ ocf_log info "NIC $NIC is DOWN" ++ exit $OCF_SUCCESS + fi + } + +@@ -1040,12 +1032,8 @@ ip_monitor() { + then + return $OCF_SUCCESS + else +- if [ "$OCF_RESKEY_dc" = "yes" ]; then +- ocf_log info "NIC $NIC is DOWN" +- return $OCF_SUCCESS +- else +- return $OCF_NOT_RUNNING +- fi ++ ocf_log info "NIC $NIC is DOWN" ++ return $OCF_SUCCESS + fi + ;; + partial|no|partial2) +-- +1.8.3.1 + From 7b86cd14dfed473780befe2e5f7d2e4749751037 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Wed, 5 Sep 2018 20:33:52 +0200 Subject: [PATCH 4/9] Add explicit noop gate job This repo defines jobs in check queue, so we can remove the noop-jobs template usage from project-config. To allow that, add a noop job to gate queue. Change-Id: Ie204c40291c17365887793bb5f114652fa24b360 --- .zuul.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.zuul.yaml b/.zuul.yaml index 549078afd..cc57f0713 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -6,3 +6,6 @@ voting: false - openstack-tox-linters: voting: false + gate: + jobs: + - noop From dba551a518f67371d95e0152a5ef2323735852ef Mon Sep 17 00:00:00 2001 From: Martin Chen Date: Wed, 5 Sep 2018 01:10:16 +0800 Subject: [PATCH 5/9] Fix linters error and enable linters zuul gate Fix below linters errors E010 The "do" should be on same line as for E010 The "do" should be on same line as while E011 Then keyword is not on same line as if or elif keyword E020 Function declaration not in format ^function name {$ Ignore: E041 Arithmetic expansion using $[ is deprecated for $(( E042 local declaration hides errors E043 Arithmetic compound has inconsistent return semantics E044 Use [[ for non-POSIX comparisions Story: 2003366 Task: 24423 Change-Id: I8b6b72e702d3e89d1813772d6bf16819e28e818c Signed-off-by: Martin Chen --- .zuul.yaml | 7 +-- base/haproxy/haproxy/haproxy.sh | 7 +-- base/initscripts/centos/files/mountnfs.sh | 9 ++-- database/mariadb/centos/files/clustercheck.sh | 3 +- .../centos/files/mysql-prepare-db-dir.sh | 15 ++---- filesystem/nfscheck/files/nfscheck.sh | 15 ++---- .../lldpd/centos/files/i40e-lldp-configure.sh | 27 ++++------ .../mlx4-config/files/mlx4-configure.sh | 18 +++---- .../mlx4-config/files/mlx4_core_goenabled.sh | 6 +-- tools/collector/scripts/collect_ceph.sh | 6 +-- tools/collector/scripts/collect_ima.sh | 3 +- tools/collector/scripts/collect_nfv_vim.sh | 3 +- tools/collector/scripts/collect_openstack.sh | 3 +- tools/collector/scripts/collect_psqldb.sh | 15 ++---- tools/collector/scripts/collect_sysinv.sh | 3 +- tools/collector/scripts/collect_tc.sh | 12 ++--- .../hostdata-collectors/scripts/ceph.sh | 6 +-- .../scripts/cleanup-engtools.sh | 15 +++--- .../scripts/collect-engtools.sh | 19 +++---- .../hostdata-collectors/scripts/diskstats.sh | 9 ++-- .../scripts/engtools_util.sh | 51 ++++++++++--------- .../hostdata-collectors/scripts/filestats.sh | 6 +-- .../scripts/init.d/collect-engtools.sh | 3 +- .../scripts/linux_benchmark.sh | 42 +++++---------- .../hostdata-collectors/scripts/memstats.sh | 6 +-- .../hostdata-collectors/scripts/netstats.sh | 9 ++-- .../hostdata-collectors/scripts/postgres.sh | 9 ++-- .../hostdata-collectors/scripts/rabbitmq.sh | 6 +-- .../scripts/remote/rsync-engtools-data.sh | 6 +-- .../hostdata-collectors/scripts/ticker.sh | 3 +- .../hostdata-collectors/scripts/vswitch.sh | 6 +-- .../parsers/common/cleanup-uncompressed.sh | 3 +- tools/engtools/parsers/common/parse-all.sh | 36 +++++-------- tools/engtools/parsers/common/parse-daily.sh | 6 +-- .../engtools/parsers/common/parse-postgres.sh | 12 ++--- .../engtools/parsers/common/parse-schedtop.sh | 9 ++-- tools/engtools/parsers/common/parse-util.sh | 9 ++-- tox.ini | 12 +++-- .../platform-util/scripts/cgcs_tc_setup.sh | 33 ++++-------- .../scripts/remotelogging_tc_setup.sh | 12 ++--- 40 files changed, 181 insertions(+), 299 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index cc57f0713..4ffd3fcba 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -4,8 +4,9 @@ jobs: - openstack-tox-pep8: voting: false - - openstack-tox-linters: - voting: false + - openstack-tox-linters gate: jobs: - - noop + - openstack-tox-pep8: + voting: false + - openstack-tox-linters diff --git a/base/haproxy/haproxy/haproxy.sh b/base/haproxy/haproxy/haproxy.sh index dc7e6ea92..8791f9a1a 100755 --- a/base/haproxy/haproxy/haproxy.sh +++ b/base/haproxy/haproxy/haproxy.sh @@ -59,7 +59,9 @@ start() { } stop() { - if [ ! -e $PIDFILE ]; then return; fi + if [ ! -e $PIDFILE ]; then + return + fi echo -n "Stopping $DESC..." @@ -73,8 +75,7 @@ stop() { remove_TPM_transients } -status() -{ +status() { pid=`cat $PIDFILE 2>/dev/null` if [ -n "$pid" ]; then if ps -p $pid &>/dev/null ; then diff --git a/base/initscripts/centos/files/mountnfs.sh b/base/initscripts/centos/files/mountnfs.sh index 8d55c9175..adeea3d00 100755 --- a/base/initscripts/centos/files/mountnfs.sh +++ b/base/initscripts/centos/files/mountnfs.sh @@ -29,8 +29,7 @@ mount_nfs=no mount_smb=no mount_ncp=no mount_cifs=no -while read device mountpt fstype options -do +while read device mountpt fstype options; do case "$device" in ""|\#*) continue @@ -70,8 +69,7 @@ done exec 0>&1 -if test "$rpcbind" = yes -then +if test "$rpcbind" = yes; then # WRL: Centos precheck: Dont start rpcbind in this init script. # It is started by a systemd service file. if test "/etc/centos-release" = no @@ -88,8 +86,7 @@ then fi fi -if test "$mount_nfs" = yes || test "$mount_smb" = yes || test "$mount_ncp" = yes || test "$mount_cifs" = yes -then +if test "$mount_nfs" = yes || test "$mount_smb" = yes || test "$mount_ncp" = yes || test "$mount_cifs" = yes; then echo "Mounting remote filesystems..." test "$mount_nfs" = yes && mount -a -t nfs test "$mount_smb" = yes && mount -a -t smbfs diff --git a/database/mariadb/centos/files/clustercheck.sh b/database/mariadb/centos/files/clustercheck.sh index 782dbb6f1..ce16a81dc 100644 --- a/database/mariadb/centos/files/clustercheck.sh +++ b/database/mariadb/centos/files/clustercheck.sh @@ -43,8 +43,7 @@ fi WSREP_STATUS=$($MYSQL_CMDLINE -e "SHOW STATUS LIKE 'wsrep_local_state';" \ 2>${ERR_FILE} | tail -1 2>>${ERR_FILE}) -if [[ "${WSREP_STATUS}" == "4" ]] || [[ "${WSREP_STATUS}" == "2" && ${AVAILABLE_WHEN_DONOR} == 1 ]] -then +if [[ "${WSREP_STATUS}" == "4" ]] || [[ "${WSREP_STATUS}" == "2" && ${AVAILABLE_WHEN_DONOR} == 1 ]]; then # Check only when set to 0 to avoid latency in response. if [[ $AVAILABLE_WHEN_READONLY -eq 0 ]];then READ_ONLY=$($MYSQL_CMDLINE -e "SHOW GLOBAL VARIABLES LIKE 'read_only';" \ diff --git a/database/mariadb/centos/files/mysql-prepare-db-dir.sh b/database/mariadb/centos/files/mysql-prepare-db-dir.sh index 96e899813..52a42a067 100644 --- a/database/mariadb/centos/files/mysql-prepare-db-dir.sh +++ b/database/mariadb/centos/files/mysql-prepare-db-dir.sh @@ -27,30 +27,26 @@ should_initialize () # If two args given first is user, second is group # otherwise the arg is the systemd service file -if [ "$#" -eq 2 ] -then +if [ "$#" -eq 2 ]; then myuser="$1" mygroup="$2" else # Absorb configuration settings from the specified systemd service file, # or the default service if not specified SERVICE_NAME="$1" - if [ x"$SERVICE_NAME" = x ] - then + if [ x"$SERVICE_NAME" = x ]; then SERVICE_NAME=@DAEMON_NAME@.service fi myuser=`systemctl show -p User "${SERVICE_NAME}" | sed 's/^User=//'` - if [ x"$myuser" = x ] - then + if [ x"$myuser" = x ]; then myuser=mysql fi mygroup=`systemctl show -p Group "${SERVICE_NAME}" | sed 's/^Group=//'` - if [ x"$mygroup" = x ] - then + if [ x"$mygroup" = x ]; then mygroup=mysql fi fi @@ -79,8 +75,7 @@ chmod 0640 "$errlogfile" if should_initialize "$datadir" ; then # First, make sure $datadir is there with correct permissions # (note: if it's not, and we're not root, this'll fail ...) - if [ ! -e "$datadir" -a ! -h "$datadir" ] - then + if [ ! -e "$datadir" -a ! -h "$datadir" ]; then mkdir -p "$datadir" || exit 1 fi chown "$myuser:$mygroup" "$datadir" diff --git a/filesystem/nfscheck/files/nfscheck.sh b/filesystem/nfscheck/files/nfscheck.sh index f58ab80ef..3b0679138 100644 --- a/filesystem/nfscheck/files/nfscheck.sh +++ b/filesystem/nfscheck/files/nfscheck.sh @@ -12,12 +12,10 @@ MOUNT=/opt/platform previous=1 delay=60 -while : -do +while : ; do # First, check that it's actually an NFS mount mount | grep -q $MOUNT - if [ $? -ne 0 ] - then + if [ $? -ne 0 ]; then logger -t NFSCHECK "$MOUNT is not mounted" previous=1 sleep $delay @@ -31,18 +29,15 @@ do # At this point, jobs will either report no jobs (empty) or Done, # unless the job is still running/hung rc=$(jobs) - if [[ -z "$rc" || $rc =~ "Done" ]] - then + if [[ -z "$rc" || $rc =~ "Done" ]]; then # NFS is successful - if [ $previous -ne 0 ] - then + if [ $previous -ne 0 ]; then logger -t NFSCHECK "NFS test of $MOUNT is ok" previous=0 fi else # Keep waiting until the job is done - while ! [[ -z "$rc" || $rc =~ "Done" ]] - do + while ! [[ -z "$rc" || $rc =~ "Done" ]]; do logger -t NFSCHECK "NFS test of $MOUNT is failed" previous=1 sleep $delay diff --git a/networking/lldpd/centos/files/i40e-lldp-configure.sh b/networking/lldpd/centos/files/i40e-lldp-configure.sh index 431a405fa..4a5e8a441 100644 --- a/networking/lldpd/centos/files/i40e-lldp-configure.sh +++ b/networking/lldpd/centos/files/i40e-lldp-configure.sh @@ -25,20 +25,17 @@ DEBUGFS_PATH=/sys/kernel/debug DEBUGFS_I40_DEVICES_PATH=$DEBUGFS_PATH/i40e LLDP_COMMAND=lldp -function log() -{ +function log { local MSG="${PROGNAME}: $1" logger -p notice "${MSG}" } -function err() -{ +function err { local MSG="${PROGNAME}: $1" logger -p error "${MSG}" } -function configure_device() -{ +function configure_device { local DEVICE=$1 local ACTION=$2 local DEVICE_PATH=${DEBUGFS_I40_DEVICES}/${DEVICE} @@ -59,23 +56,22 @@ function configure_device() return ${RET} } -function is_debugfs_mounted() { +function is_debugfs_mounted { if grep -qs "${DEBUGFS_PATH}" /proc/mounts; then return 0 fi return 1 } -function mount_debugfs() { +function mount_debugfs { mount -t debugfs none ${DEBUGFS_PATH} } -function unmount_debugfs() { +function unmount_debugfs { umount ${DEBUGFS_PATH} } -function scan_devices() -{ +function scan_devices { local ACTION=$1 local DEBUGFS_MOUNTED="false" local DEVICES=${DEBUGFS_I40_DEVICES_PATH}/* @@ -111,20 +107,17 @@ function scan_devices() return 0 } -function start() -{ +function start { scan_devices start return $? } -function stop() -{ +function stop { scan_devices stop return $? } -function status() -{ +function status { return 0 } diff --git a/networking/mellanox/mlx4-config/files/mlx4-configure.sh b/networking/mellanox/mlx4-config/files/mlx4-configure.sh index 0005ce2e4..959024e09 100644 --- a/networking/mellanox/mlx4-config/files/mlx4-configure.sh +++ b/networking/mellanox/mlx4-config/files/mlx4-configure.sh @@ -15,8 +15,7 @@ DEBUG=${DEBUG:-0} # all files beginning in "mlx4_port" and ending in one or more digits. shopt -s extglob -function log() -{ +function log { local MSG="${PROGNAME}: $1" if [ ${DEBUG} -ne 0 ]; then echo "${MSG}" @@ -24,8 +23,7 @@ function log() echo "${MSG}" >> /var/log/mlx4-configure.log } -function configure_device() -{ +function configure_device { local DEVICE=$1 local DEVICE_PATH=${SYSFS_PCI_DEVICES}/${DEVICE} @@ -58,8 +56,7 @@ function configure_device() } -function scan_devices() -{ +function scan_devices { local DEVICES=$(ls -1 ${SYSFS_PCI_DEVICES}) for DEVICE in ${DEVICES}; do @@ -83,19 +80,16 @@ function scan_devices() } -function start() -{ +function start { scan_devices return $? } -function stop() -{ +function stop { return 0 } -function status() -{ +function status { return 0 } diff --git a/networking/mellanox/mlx4-config/files/mlx4_core_goenabled.sh b/networking/mellanox/mlx4-config/files/mlx4_core_goenabled.sh index c1a3592d7..189cf3214 100644 --- a/networking/mellanox/mlx4-config/files/mlx4_core_goenabled.sh +++ b/networking/mellanox/mlx4-config/files/mlx4_core_goenabled.sh @@ -12,13 +12,11 @@ NAME=$(basename $0) OPTIONS_CHANGED_FLAG=/var/run/.mlx4_cx3_reboot_required COMPUTE_CONFIG_COMPLETE=/var/run/.compute_config_complete -function LOG() -{ +function LOG { logger "$NAME: $*" } -if [ -f $OPTIONS_CHANGED_FLAG ] && [ -f $COMPUTE_CONFIG_COMPLETE ] -then +if [ -f $OPTIONS_CHANGED_FLAG ] && [ -f $COMPUTE_CONFIG_COMPLETE ]; then LOG "mlx4_core options has been changed. Failing goenabled check." exit 1 fi diff --git a/tools/collector/scripts/collect_ceph.sh b/tools/collector/scripts/collect_ceph.sh index 0898d8db8..1a5863ed1 100755 --- a/tools/collector/scripts/collect_ceph.sh +++ b/tools/collector/scripts/collect_ceph.sh @@ -14,8 +14,7 @@ SERVICE="ceph" LOGFILE="${extradir}/ceph.info" echo "${hostname}: Ceph Info .........: ${LOGFILE}" -function is_service_active() -{ +function is_service_active { active=`sm-query service management-ip | grep "enabled-active"` if [ -z "$active" ] ; then return 0 @@ -24,8 +23,7 @@ function is_service_active() fi } -function exit_if_timeout() -{ +function exit_if_timeout { if [ "$?" = "124" ] ; then echo "Exiting due to ceph command timeout" >> ${LOGFILE} exit 0 diff --git a/tools/collector/scripts/collect_ima.sh b/tools/collector/scripts/collect_ima.sh index dd35bfa63..14c751e42 100755 --- a/tools/collector/scripts/collect_ima.sh +++ b/tools/collector/scripts/collect_ima.sh @@ -10,8 +10,7 @@ source /usr/local/sbin/collect_parms source /usr/local/sbin/collect_utils -function is_extended_profile() -{ +function is_extended_profile { if [ ! -n "${security_profile}" ] || [ "${security_profile}" != "extended" ]; then return 0 else diff --git a/tools/collector/scripts/collect_nfv_vim.sh b/tools/collector/scripts/collect_nfv_vim.sh index 4b9928135..c5ccbc7fa 100644 --- a/tools/collector/scripts/collect_nfv_vim.sh +++ b/tools/collector/scripts/collect_nfv_vim.sh @@ -12,8 +12,7 @@ source /usr/local/sbin/collect_utils LOGFILE="${extradir}/nfv-vim.info" echo "${hostname}: NFV-Vim Info ......: ${LOGFILE}" -function is_service_active() -{ +function is_service_active { active=`sm-query service vim | grep "enabled-active"` if [ -z "$active" ] ; then return 0 diff --git a/tools/collector/scripts/collect_openstack.sh b/tools/collector/scripts/collect_openstack.sh index d4e986b62..42d7d24d8 100755 --- a/tools/collector/scripts/collect_openstack.sh +++ b/tools/collector/scripts/collect_openstack.sh @@ -10,8 +10,7 @@ source /usr/local/sbin/collect_parms source /usr/local/sbin/collect_utils -function is_service_active() -{ +function is_service_active { active=`sm-query service rabbit-fs | grep "enabled-active"` if [ -z "$active" ] ; then return 0 diff --git a/tools/collector/scripts/collect_psqldb.sh b/tools/collector/scripts/collect_psqldb.sh index f6613b60c..d223b1b7b 100755 --- a/tools/collector/scripts/collect_psqldb.sh +++ b/tools/collector/scripts/collect_psqldb.sh @@ -19,8 +19,7 @@ DB_DIR="${extradir}/database" LOGFILE="${extradir}/database.info" echo "${hostname}: Database Info .....: ${LOGFILE}" -function is_service_active() -{ +function is_service_active { active=`sm-query service postgres | grep "enabled-active"` if [ -z "$active" ] ; then return 0 @@ -34,11 +33,9 @@ function is_service_active() ############################################################################### mkdir -p ${DB_DIR} -function log_database() -{ +function log_database { db_list=( $(${PSQL_CMD} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) - for db in "${db_list[@]}" - do + for db in "${db_list[@]}"; do echo "postgres database: ${db}" ${PSQL_CMD} -d ${db} -c " SELECT @@ -75,12 +72,10 @@ function log_database() DB_EXT=db.sql.txt -function database_dump() -{ +function database_dump { mkdir -p ${DB_DIR} db_list=( $(${PSQL_CMD} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;") ) - for DB in "${db_list[@]}" - do + for DB in "${db_list[@]}"; do if [ "$DB" != "keystone" -a "$DB" != "ceilometer" ] ; then echo "${hostname}: Dumping Database ..: ${DB_DIR}/$DB.$DB_EXT" (cd ${DB_DIR} ; sudo -u postgres pg_dump $DB > $DB.$DB_EXT) diff --git a/tools/collector/scripts/collect_sysinv.sh b/tools/collector/scripts/collect_sysinv.sh index 37ba26af5..3f27a2886 100755 --- a/tools/collector/scripts/collect_sysinv.sh +++ b/tools/collector/scripts/collect_sysinv.sh @@ -14,8 +14,7 @@ SERVICE="inventory" LOGFILE="${extradir}/${SERVICE}.info" RPMLOG="${extradir}/rpm.info" -function is_service_active() -{ +function is_service_active { active=`sm-query service management-ip | grep "enabled-active"` if [ -z "$active" ] ; then return 0 diff --git a/tools/collector/scripts/collect_tc.sh b/tools/collector/scripts/collect_tc.sh index 160b970e6..737461a88 100755 --- a/tools/collector/scripts/collect_tc.sh +++ b/tools/collector/scripts/collect_tc.sh @@ -27,8 +27,8 @@ fi delimiter ${LOGFILE} "ip link" ip link >> ${LOGFILE} -for i in $(ip link | grep mtu | grep eth |awk '{print $2}' | sed 's#:##g'); -do +for i in $(ip link | grep mtu | grep eth |awk '{print $2}' | sed 's#:##g'); do + delimiter ${LOGFILE} "ethtool ${i}" ethtool ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} @@ -55,8 +55,8 @@ fi delimiter ${LOGFILE} "tc qdisc show" tc qdisc show >> ${LOGFILE} -for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); -do +for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); do + delimiter ${LOGFILE} "tc class show dev ${i}" tc class show dev ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} @@ -70,8 +70,8 @@ done delimiter ${LOGFILE} "tc -s qdisc show" tc -s qdisc show >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} -for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); -do +for i in $(ip link | grep htb | awk '{print $2}' | sed 's#:##g'); do + delimiter ${LOGFILE} "tc -s class show dev ${i}" tc -s class show dev ${i} >> ${LOGFILE} 2>>${COLLECT_ERROR_LOG} diff --git a/tools/engtools/hostdata-collectors/scripts/ceph.sh b/tools/engtools/hostdata-collectors/scripts/ceph.sh index f05904ffc..61e5540d7 100644 --- a/tools/engtools/hostdata-collectors/scripts/ceph.sh +++ b/tools/engtools/hostdata-collectors/scripts/ceph.sh @@ -14,8 +14,7 @@ fi OPT_USE_INTERVALS=1 # Print key ceph statistics -function print_ceph() -{ +function print_ceph { print_separator TOOL_HIRES_TIME @@ -47,8 +46,7 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do +for ((rep=1; rep <= REPEATS ; rep++)); do print_ceph sleep ${INTERVAL_SEC} done diff --git a/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh b/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh index 8fc7b8b1e..5d5701b53 100644 --- a/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/cleanup-engtools.sh @@ -32,14 +32,12 @@ LOG "Cleanup engtools:" # ( be sure not to clobber /etc/init.d/collect-engtools.sh ) LOG "kill processes brute force" pids=( $(pidof -x /usr/local/bin/collect-engtools.sh) ) -if [ ${#pids[@]} -ne 0 ] -then +if [ ${#pids[@]} -ne 0 ]; then LOG "killing: ${pids[@]}" - for pid in ${pids[@]} - do - LOG "kill: [ ${pid} ] " - pkill -KILL -P ${pid} - kill -9 ${pid} + for pid in ${pids[@]}; do + LOG "kill: [ ${pid} ] " + pkill -KILL -P ${pid} + kill -9 ${pid} done pkill -KILL iostat pkill -KILL top @@ -48,8 +46,7 @@ else fi LOG "remove pidfiles" -for TOOL in "${TOOLS[@]}" -do +for TOOL in "${TOOLS[@]}"; do rm -f -v /var/run/${TOOL}.pid done LOG "done" diff --git a/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh b/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh index 5e3c06430..e59f382b4 100644 --- a/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/collect-engtools.sh @@ -48,11 +48,9 @@ declare tgt_avail_bytes declare tgt_used_bytes # do_parallel_commands - launch parallel tools with separate output files -function do_parallel_commands() -{ +function do_parallel_commands { parallel_outfiles=() - for elem in "${tlist[@]}" - do + for elem in "${tlist[@]}"; do tool=""; period=""; repeat=""; interval="" my_hash="elem[*]" local ${!my_hash} @@ -75,8 +73,7 @@ function do_parallel_commands() # get_current_avail_usage() - get output destination file-system usage and # availability. # - updates: df_size_bytes, df_avail_bytes, du_used_bytes -function get_current_avail_usage() -{ +function get_current_avail_usage { local -a df_arr_bytes=( $(df -P --block-size=1 ${TOOL_DEST_DIR} | awk 'NR==2 {print $2, $4}') ) df_size_bytes=${df_arr_bytes[0]} df_avail_bytes=${df_arr_bytes[1]} @@ -85,8 +82,7 @@ function get_current_avail_usage() # purge_oldest_files() - remove oldest files based on file-system available space, # and maximum collection size -function purge_oldest_files() -{ +function purge_oldest_files { # get current file-system usage get_current_avail_usage msg=$(printf "avail %d MB, headroom %d MB; used %d MB, max %d MB" \ @@ -116,8 +112,7 @@ function purge_oldest_files() # remove files in oldest time sorted order until we meet usage targets, # incrementally updating usage as we remve files - for file in $( ls -rt ${TOOL_DEST_DIR}/${HOSTNAME}_* 2>/dev/null ) - do + for file in $( ls -rt ${TOOL_DEST_DIR}/${HOSTNAME}_* 2>/dev/null ); do if [[ $df_avail_bytes -ge $tgt_avail_bytes ]] && \ [[ $du_used_bytes -le $tgt_used_bytes ]]; then break @@ -307,9 +302,7 @@ REP=0 if [ ${#tlist[@]} -ne 0 ]; then # Static stats collection is turned on - while [[ ${TOOL_USR1_SIGNAL} -eq 0 ]] && - [[ ${OPT_FOREVER} -eq 1 || ${REP} -lt ${REPEATS} ]] - do + while [[ ${TOOL_USR1_SIGNAL} -eq 0 ]] && [[ ${OPT_FOREVER} -eq 1 || ${REP} -lt ${REPEATS} ]]; do # increment loop counter ((REP++)) diff --git a/tools/engtools/hostdata-collectors/scripts/diskstats.sh b/tools/engtools/hostdata-collectors/scripts/diskstats.sh index d00d777ba..c419a9a0e 100644 --- a/tools/engtools/hostdata-collectors/scripts/diskstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/diskstats.sh @@ -14,8 +14,7 @@ fi OPT_USE_INTERVALS=1 # Print disk summary -function print_disk() -{ +function print_disk { print_separator TOOL_HIRES_TIME @@ -57,8 +56,7 @@ function print_disk() } # Print disk static summary -function print_disk_static() -{ +function print_disk_static { print_separator cmd='cat /proc/scsi/scsi' ${ECHO} "Attached devices: ${cmd}" @@ -109,8 +107,7 @@ print_disk_static # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do +for ((rep=1; rep <= REPEATS ; rep++)); do print_disk sleep ${INTERVAL_SEC} done diff --git a/tools/engtools/hostdata-collectors/scripts/engtools_util.sh b/tools/engtools/hostdata-collectors/scripts/engtools_util.sh index 4ac66a9ae..af1aaa4e4 100644 --- a/tools/engtools/hostdata-collectors/scripts/engtools_util.sh +++ b/tools/engtools/hostdata-collectors/scripts/engtools_util.sh @@ -24,8 +24,7 @@ else . /etc/init.d/functions fi # Lightweight replacement for pidofproc -p -function check_pidfile () -{ +function check_pidfile { local pidfile pid OPTIND=1 @@ -53,8 +52,7 @@ function check_pidfile () } # tools_init - initialize tool resources -function tools_init () -{ +function tools_init { local rc=0 local error=0 TOOLNAME=$(basename $0) @@ -199,7 +197,7 @@ function tools_init () } # tools_cleanup() - terminate child processes -function tools_cleanup() { +function tools_cleanup { # restore signal handling to default behaviour trap - INT HUP TERM EXIT trap - USR1 USR2 @@ -230,43 +228,47 @@ function tools_cleanup() { } # tools_exit_handler() - exit handler routine -function tools_exit_handler() { +function tools_exit_handler { TOOL_EXIT_SIGNAL=1 tools_cleanup 128 } # tools_usr1_handler() - USR1 handler routine -function tools_usr1_handler() { +function tools_usr1_handler { TOOL_USR1_SIGNAL=1 LOG "caught USR1" } # tools_usr2_handler() - USR2 handler routine -function tools_usr2_handler() { +function tools_usr2_handler { TOOL_USR2_SIGNAL=1 LOG "caught USR1" } # LOG(), WARNLOG(), ERRLOG() - simple print log functions (not logger) -function LOG () -{ +function LOG { local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) echo "${tstamp_H} ${HOSTNAME} $0($$): $@"; } -function LOG_NOCR () -{ + +function LOG_NOCR { local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) echo -n "${tstamp_H} ${HOSTNAME} $0($$): $@"; } -function WARNLOG () { LOG "WARN $@"; } -function ERRLOG () { LOG "ERROR $@"; } + +function WARNLOG { + LOG "WARN $@"; +} + +function ERRLOG { + LOG "ERROR $@"; +} # TOOL_HIRES_TIME() - easily parsed date/timestamp and hi-resolution uptime -function TOOL_HIRES_TIME() -{ +function TOOL_HIRES_TIME { echo "time: " $( ${DATE} +"%a %F %H:%M:%S.%N %Z %z" ) "uptime: " $( cat /proc/uptime ) } # set_affinity() - set affinity for current script if a a CPULIST is defined -function set_affinity() { +function set_affinity { local CPULIST=$1 if [ -z "${CPULIST}" ]; then return @@ -280,7 +282,7 @@ function set_affinity() { } # cmd_idle_priority() - command to set nice + ionice -function cmd_idle_priority() { +function cmd_idle_priority { local NICE="" local IONICE="" @@ -301,13 +303,13 @@ function cmd_idle_priority() { # print_separator() - print a horizontal separation line '\u002d' is '-' -function print_separator () { +function print_separator { printf '\u002d%.s' {1..80} printf '\n' } # tools_header() - print out common GenWare tools header -function tools_header() { +function tools_header { local TOOLNAME=$(basename $0) # Get timestamp @@ -393,7 +395,7 @@ function tools_header() { # tools_usage() - show generic tools tool usage -function tools_usage() { +function tools_usage { if [ ${OPT_USE_INTERVALS} -eq 1 ]; then echo "usage: ${TOOLNAME} [-f] [-p ] [-i ] [-c ] [-h]" else @@ -402,7 +404,7 @@ function tools_usage() { } # tools_print_help() - print generic tool help -function tools_print_help() { +function tools_print_help { tools_usage echo echo "Options:"; @@ -423,7 +425,7 @@ function tools_print_help() { } # tools_parse_options() -- parse common options for tools scripts -function tools_parse_options() { +function tools_parse_options { # check for no arguments, print usage if [ $# -eq "0" ]; then tools_usage @@ -432,8 +434,7 @@ function tools_parse_options() { fi # parse the input arguments - while getopts "fp:i:c:h" Option - do + while getopts "fp:i:c:h" Option; do case $Option in f) OPT_FOREVER=1 diff --git a/tools/engtools/hostdata-collectors/scripts/filestats.sh b/tools/engtools/hostdata-collectors/scripts/filestats.sh index 4f06b7562..8b38e695b 100644 --- a/tools/engtools/hostdata-collectors/scripts/filestats.sh +++ b/tools/engtools/hostdata-collectors/scripts/filestats.sh @@ -16,8 +16,7 @@ PAGE_SIZE=$(getconf PAGE_SIZE) OPT_USE_INTERVALS=1 -function print_files() -{ +function print_files { print_separator TOOL_HIRES_TIME @@ -85,8 +84,7 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do +for ((rep=1; rep <= REPEATS ; rep++)); do print_files sleep ${INTERVAL_SEC} done diff --git a/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh b/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh index f4c93ff67..fa52a65a6 100644 --- a/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh +++ b/tools/engtools/hostdata-collectors/scripts/init.d/collect-engtools.sh @@ -69,8 +69,7 @@ case $1 in stop) if [ -e ${PIDFILE} ]; then pids=$(pidof -x ${NAME}) - if [[ ! -z "${pids}" ]] - then + if [[ ! -z "${pids}" ]]; then echo_success "Stopping ${NAME} [$pid]" start-stop-daemon --stop --quiet --oknodo --pidfile ${PIDFILE} --retry=TERM/3/KILL/5 # [ JGAULD: none of the following should be necessary ] diff --git a/tools/engtools/hostdata-collectors/scripts/linux_benchmark.sh b/tools/engtools/hostdata-collectors/scripts/linux_benchmark.sh index fb1d16d47..64fd5ce8b 100644 --- a/tools/engtools/hostdata-collectors/scripts/linux_benchmark.sh +++ b/tools/engtools/hostdata-collectors/scripts/linux_benchmark.sh @@ -29,8 +29,7 @@ compute_ports=(8000 8001 8002) traffic_types=(storage migration default drbd) flow_ids=(1:20 1:30 1:40 1:50) -function exec_cmd () -{ +function exec_cmd { node="$1" cmd="$2" @@ -41,8 +40,7 @@ function exec_cmd () fi } -function iperf3_server_start () -{ +function iperf3_server_start { local server="$1" local result="$2" local port="$3" @@ -55,8 +53,7 @@ function iperf3_server_start () $(exec_cmd "${server}" "${cmd}") } -function iperf3_client_tcp_start () -{ +function iperf3_client_tcp_start { local result="${result_dir}/throughput" local cmd="" local client="$1" @@ -76,8 +73,7 @@ function iperf3_client_tcp_start () $(exec_cmd "${client}" "${cmd} > ${result} 2>&1") } -function iperf3_client_udp_start () -{ +function iperf3_client_udp_start { local result="${result_dir}/throughput_udp" local cmd="" local client="$1" @@ -102,20 +98,17 @@ function iperf3_client_udp_start () $(exec_cmd "${client}" "${cmd} -b ${bw} >> ${result} 2>&1" ) } -function iperf3_stop () -{ +function iperf3_stop { local node="$1" local cmd="pkill iperf3" $(exec_cmd "${node}" "${cmd}") } -function get_ip_addr () -{ +function get_ip_addr { arp -a | grep -oP "(?<=$1 \()[^)]*" | head -n 1 } -function throughput_tcp_test() -{ +function throughput_tcp_test { for (( i = 0; i < ${#nodes[@]} ; i+=2 )); do for interface in "${interfaces[@]}"; do local interface_name="management" @@ -137,8 +130,7 @@ function throughput_tcp_test() done } -function throughput_udp_test () -{ +function throughput_udp_test { for (( i = 0; i < ${#nodes[@]} ; i+=2 )); do for interface in "${interfaces[@]}"; do local interface_name="management" @@ -229,8 +221,7 @@ function throughput_udp_test () done } -function throughput_parallel_test () -{ +function throughput_parallel_test { local dev="" local ip_addr="" local interface_name="" @@ -304,8 +295,7 @@ function throughput_parallel_test () done } -function latency_test () -{ +function latency_test { for (( i = 0; i < ${#nodes[@]} ; i+=2 )); do for interface in "${interfaces[@]}"; do local interface_name="management" @@ -328,16 +318,14 @@ function latency_test () done } -function setup () -{ +function setup { for node in ${nodes[@]}; do iperf3_stop "${node}" $(exec_cmd "${node}" "rm -rf ${result_dir}; mkdir -p ${result_dir}") done } -function get_remote_results () -{ +function get_remote_results { for node in ${nodes[@]}; do if [ "${node}" != "${host}" ]; then mkdir ${result_dir}/${node} @@ -346,8 +334,7 @@ function get_remote_results () done } -function get_interface_info () -{ +function get_interface_info { local dev="" local ip_addr="" printf "Network interfaces info\n" >> ${summary_file} @@ -365,8 +352,7 @@ function get_interface_info () done } -function generate_summary () -{ +function generate_summary { local header="" local result="" local result_file="" diff --git a/tools/engtools/hostdata-collectors/scripts/memstats.sh b/tools/engtools/hostdata-collectors/scripts/memstats.sh index 81e47777d..5ba340f0a 100644 --- a/tools/engtools/hostdata-collectors/scripts/memstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/memstats.sh @@ -16,8 +16,7 @@ PAGE_SIZE=$(getconf PAGE_SIZE) OPT_USE_INTERVALS=1 # Print key networking device statistics -function print_memory() -{ +function print_memory { # Configuration for netcmds MEMINFO=/proc/meminfo NODEINFO=/sys/devices/system/node/node?/meminfo @@ -99,8 +98,7 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do +for ((rep=1; rep <= REPEATS ; rep++)); do print_memory sleep ${INTERVAL_SEC} done diff --git a/tools/engtools/hostdata-collectors/scripts/netstats.sh b/tools/engtools/hostdata-collectors/scripts/netstats.sh index cd306f743..c9be0a1e5 100644 --- a/tools/engtools/hostdata-collectors/scripts/netstats.sh +++ b/tools/engtools/hostdata-collectors/scripts/netstats.sh @@ -14,8 +14,7 @@ fi OPT_USE_INTERVALS=1 # Print key networking device statistics -function print_netcmds() -{ +function print_netcmds { # Configuration for netcmds DEV=/proc/net/dev NETSTAT=/proc/net/netstat @@ -26,8 +25,7 @@ function print_netcmds() for net in \ ${DEV} ${NETSTAT} do - if [ -e "${net}" ] - then + if [ -e "${net}" ]; then ${ECHO} "# ${net}" ${CAT} ${net} ${ECHO} @@ -53,8 +51,7 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do +for ((rep=1; rep <= REPEATS ; rep++)); do print_netcmds sleep ${INTERVAL_SEC} done diff --git a/tools/engtools/hostdata-collectors/scripts/postgres.sh b/tools/engtools/hostdata-collectors/scripts/postgres.sh index 7ee0a8577..340d0e429 100644 --- a/tools/engtools/hostdata-collectors/scripts/postgres.sh +++ b/tools/engtools/hostdata-collectors/scripts/postgres.sh @@ -14,8 +14,7 @@ fi OPT_USE_INTERVALS=1 # Print key networking device statistics -function print_postgres() -{ +function print_postgres { print_separator TOOL_HIRES_TIME @@ -40,8 +39,7 @@ ORDER BY pg_database_size DESC; " # For each database, list tables and their sizes (similar to "\dt+") - for db in "${db_list[@]}" - do + for db in "${db_list[@]}"; do ${ECHO} "# postgres database: ${db}" ${PSQL} -d ${db} -c " SELECT @@ -128,8 +126,7 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do +for ((rep=1; rep <= REPEATS ; rep++)); do print_postgres sleep ${INTERVAL_SEC} done diff --git a/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh b/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh index 64e0de9cb..2755024f6 100644 --- a/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh +++ b/tools/engtools/hostdata-collectors/scripts/rabbitmq.sh @@ -15,8 +15,7 @@ OPT_USE_INTERVALS=1 #Need this workaround MQOPT="-n rabbit@localhost" # Print key networking device statistics -function print_rabbitmq() -{ +function print_rabbitmq { print_separator TOOL_HIRES_TIME @@ -72,8 +71,7 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do +for ((rep=1; rep <= REPEATS ; rep++)); do print_rabbitmq sleep ${INTERVAL_SEC} done diff --git a/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh b/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh index 2e6704293..095c5ddb7 100644 --- a/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh +++ b/tools/engtools/hostdata-collectors/scripts/remote/rsync-engtools-data.sh @@ -41,8 +41,7 @@ LOG "rsync engtools data from all blades:" # controllers SRC=/scratch/syseng_data/ DEST=/opt/backups/syseng_data/ -for HOST in ${CONTROLLER[@]} -do +for HOST in ${CONTROLLER[@]}; do ping -c1 ${HOST} 1>/dev/null 2>/dev/null if [ $? -eq 0 ]; then LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" @@ -55,8 +54,7 @@ done # computes & storage SRC=/tmp/syseng_data/ DEST=/opt/backups/syseng_data/ -for HOST in ${STORAGE[@]} ${COMPUTE[@]} -do +for HOST in ${STORAGE[@]} ${COMPUTE[@]}; do ping -c1 ${HOST} 1>/dev/null 2>/dev/null if [ $? -eq 0 ]; then LOG "rsync ${RSYNC_OPT} ${USER}@${HOST}:${SRC} ${DEST}" diff --git a/tools/engtools/hostdata-collectors/scripts/ticker.sh b/tools/engtools/hostdata-collectors/scripts/ticker.sh index 0bfe8ad68..1edf00ed5 100644 --- a/tools/engtools/hostdata-collectors/scripts/ticker.sh +++ b/tools/engtools/hostdata-collectors/scripts/ticker.sh @@ -29,8 +29,7 @@ set_affinity ${CPULIST} LOG_NOCR "collecting " t=0 -for ((rep=1; rep <= REPEATS ; rep++)) -do +for ((rep=1; rep <= REPEATS ; rep++)); do ((t++)) sleep ${INTERVAL_SEC} if [ ${t} -ge ${REP_LOG} ]; then diff --git a/tools/engtools/hostdata-collectors/scripts/vswitch.sh b/tools/engtools/hostdata-collectors/scripts/vswitch.sh index 1e6a5a338..f0bad0eeb 100644 --- a/tools/engtools/hostdata-collectors/scripts/vswitch.sh +++ b/tools/engtools/hostdata-collectors/scripts/vswitch.sh @@ -14,8 +14,7 @@ fi OPT_USE_INTERVALS=1 # Print key networking device statistics -function print_vswitch() -{ +function print_vswitch { print_separator TOOL_HIRES_TIME @@ -55,8 +54,7 @@ tools_header # Calculate number of sample repeats based on overall interval and sampling interval ((REPEATS = PERIOD_MIN * 60 / INTERVAL_SEC)) -for ((rep=1; rep <= REPEATS ; rep++)) -do +for ((rep=1; rep <= REPEATS ; rep++)); do print_vswitch sleep ${INTERVAL_SEC} done diff --git a/tools/engtools/parsers/common/cleanup-uncompressed.sh b/tools/engtools/parsers/common/cleanup-uncompressed.sh index 26824b269..b810fe64f 100755 --- a/tools/engtools/parsers/common/cleanup-uncompressed.sh +++ b/tools/engtools/parsers/common/cleanup-uncompressed.sh @@ -22,8 +22,7 @@ YEAR=`date +'%Y'` files="${FILE_LIST// /, }" read -p "Are you sure you want to remove all uncompressed $files files? [Y/N]: " -n 1 -r echo -if [[ $REPLY =~ ^[Y]$ ]] -then +if [[ $REPLY =~ ^[Y]$ ]]; then for FILE in ${FILE_LIST}; do rm -v */*_${YEAR}-*${FILE} done diff --git a/tools/engtools/parsers/common/parse-all.sh b/tools/engtools/parsers/common/parse-all.sh index 42f732dcc..754079693 100755 --- a/tools/engtools/parsers/common/parse-all.sh +++ b/tools/engtools/parsers/common/parse-all.sh @@ -41,8 +41,7 @@ NODE=$1 CURDATE=$(date) DATESTAMP=$(date +%b-%d) -function sedit() -{ +function sedit { local FILETOSED=$1 sed -i -e "s/ */ /g" ${FILETOSED} sed -i -e "s/ /,/g" ${FILETOSED} @@ -50,8 +49,7 @@ function sedit() sed -i "s/,$//" ${FILETOSED} } -function get_filename_from_mountname() -{ +function get_filename_from_mountname { local name=$1 local fname if test "${name#*"scratch"}" != "${name}"; then @@ -82,8 +80,7 @@ function get_filename_from_mountname() echo $fname } -function parse_process_schedtop_data() -{ +function parse_process_schedtop_data { # Logic has been moved to a separate script so that parsing process level schedtop # can be run either as part of parse-all.sh script or independently. LOG "Process level schedtop parsing is turned on in lab.conf. Parsing schedtop detail..." @@ -92,8 +89,7 @@ function parse_process_schedtop_data() cd ${NODE} } -function parse_controller_specific() -{ +function parse_controller_specific { # Parsing Postgres data, removing data from previous run if there are any. Generate summary # data for each database and detail data for specified tables LOG "Parsing postgres data for ${NODE}" @@ -123,14 +119,12 @@ function parse_controller_specific() done } -function parse_compute_specific() -{ +function parse_compute_specific { LOG "Parsing vswitch data for ${NODE}" ../parse-vswitch.sh ${NODE} } -function parse_occtop_data() -{ +function parse_occtop_data { LOG "Parsing occtop data for ${NODE}" bzcat *occtop.bz2 >occtop-${NODE}-${DATESTAMP}.txt cp occtop-${NODE}-${DATESTAMP}.txt tmp.txt @@ -181,8 +175,7 @@ function parse_occtop_data() rm tmp.txt tmp2.txt tmpdate.txt tmpcore.txt } -function parse_memtop_data() -{ +function parse_memtop_data { LOG "Parsing memtop data for ${NODE}" bzcat *memtop.bz2 > memtop-${NODE}-${DATESTAMP}.txt cp memtop-${NODE}-${DATESTAMP}.txt tmp.txt @@ -200,8 +193,7 @@ function parse_memtop_data() rm tmp.txt tmp2.txt } -function parse_netstats_data() -{ +function parse_netstats_data { LOG "Parsing netstats data for ${NODE}" # First generate the summary data then detail data for specified interfaces ../parse_netstats *netstats.bz2 > netstats-summary-${NODE}-${DATESTAMP}.txt @@ -225,8 +217,7 @@ function parse_netstats_data() fi } -function parse_iostats_data() -{ +function parse_iostats_data { LOG "Parsing iostat data for ${NODE}" if [ -z "${IOSTATS_DEVICE_LIST}" ]; then ERRLOG "IOSTAT_DEVICE_LIST is not set in host.conf. Skipping iostats..." @@ -236,8 +227,7 @@ function parse_iostats_data() echo "Date/Time,${DEVICE},rqm/s,wrqm/s,r/s,w/s,rkB/s,wkB/s,avgrq-sz,avgqu-sz,await,r_await,w_await,svctm,%util" > iostat-${NODE}-${DEVICE}.csv # Dumping iostat content to tmp file bzcat *iostat.bz2 | grep -E "/2015|/2016|/2017|${DEVICE}" | awk '{print $1","$2","$3","$4","$5","$6","$7","$8","$9","$10","$11","$12","$13","$14}' > tmp.txt - while IFS= read -r current - do + while IFS= read -r current; do if test "${current#*Linux}" != "$current" then # Skip the line that contains the word "Linux" @@ -277,8 +267,7 @@ function parse_iostats_data() fi } -function parse_diskstats_data() -{ +function parse_diskstats_data { LOG "Parsing diskstats data for ${NODE}" if [ -z "${DISKSTATS_FILESYSTEM_LIST}" ]; then @@ -362,8 +351,7 @@ if test "${NODE#*"controller"}" != "${NODE}"; then # is to use inotify which requires another inotify-tools package. oldsize=0 newsize=0 - while true - do + while true; do newsize=$(stat -c %s postgres-conns.csv) if [ "$oldsize" == "$newsize" ]; then break diff --git a/tools/engtools/parsers/common/parse-daily.sh b/tools/engtools/parsers/common/parse-daily.sh index 971c5acdd..b8c70a5fe 100755 --- a/tools/engtools/parsers/common/parse-daily.sh +++ b/tools/engtools/parsers/common/parse-daily.sh @@ -19,16 +19,14 @@ # e.g. >./parse-daily.sh memstats sm-eru # >./parse-daily.sh controller-0 filestats postgress -function print_usage() -{ +function print_usage { echo "Usage: ./parse-daily.sh will parse daily data for all hosts." echo "Usage: ./parse-daily.sh will parse daily data for specified host." echo "Valid parsers for daily stats are: memstats & filestats." exit 1 } -function parse_daily_stats() -{ +function parse_daily_stats { local PARSER_NAME=$1 local PROCESS_NAME=$2 local TMPFILE="tmp.txt" diff --git a/tools/engtools/parsers/common/parse-postgres.sh b/tools/engtools/parsers/common/parse-postgres.sh index 0d1a68d36..982412336 100755 --- a/tools/engtools/parsers/common/parse-postgres.sh +++ b/tools/engtools/parsers/common/parse-postgres.sh @@ -25,8 +25,7 @@ else fi LOG "Parsing postgres files ${FILES}" -function generate_header() -{ +function generate_header { local header="Date/Time,Total" for DB in ${DATABASE_LIST}; do if [ ${DB} == "nova_api" ]; then @@ -59,8 +58,7 @@ function generate_header() echo $header } -function generate_grep_str() -{ +function generate_grep_str { local grepstr="time:" for DB in ${DATABASE_LIST}; do grepstr="${grepstr}|${DB}" @@ -69,8 +67,7 @@ function generate_grep_str() echo $grepstr } -function init_variables() -{ +function init_variables { CONN_TOTAL="0" CONN_ACTIVE_TOTAL="0" CONN_IDLE_TOTAL="0" @@ -85,8 +82,7 @@ function init_variables() done } -function output_values() -{ +function output_values { local result="${DATEVAL} ${TIMEVAL},${CONN_TOTAL}" for DB in ${DATABASE_LIST}; do val=$(eval echo \${CONN_${DB^^}}) diff --git a/tools/engtools/parsers/common/parse-schedtop.sh b/tools/engtools/parsers/common/parse-schedtop.sh index cba3fcbba..cd70586d4 100755 --- a/tools/engtools/parsers/common/parse-schedtop.sh +++ b/tools/engtools/parsers/common/parse-schedtop.sh @@ -17,16 +17,14 @@ PARSERDIR=$(dirname $0) . ${PARSERDIR}/parse-util.sh -function print_usage() -{ +function print_usage { echo "Usage: ./parse-schedtop.sh " echo " ./parse-schedtop.sh " echo "e.g. >./parse-schedtop.sh controller-0 nova-conductor" exit 1 } -function sedit() -{ +function sedit { local FILETOSED=$1 sed -i -e "s/ */ /g" ${FILETOSED} sed -i -e "s/ /,/2g" ${FILETOSED} @@ -34,8 +32,7 @@ function sedit() sed -i "s/,$//" ${FILETOSED} } -function parse_schedtop_data() -{ +function parse_schedtop_data { HOST=$1 SERVICE=$2 LOG "Parsing ${SERVICE} schedtop for host ${HOST}" diff --git a/tools/engtools/parsers/common/parse-util.sh b/tools/engtools/parsers/common/parse-util.sh index c469d1502..907da1ee7 100755 --- a/tools/engtools/parsers/common/parse-util.sh +++ b/tools/engtools/parsers/common/parse-util.sh @@ -8,19 +8,16 @@ #LOGFILE="${PARSERDIR}/parserlog.txt" LOGFILE="parserlog.txt" -function LOG () -{ +function LOG { local tstamp_H=$( date +"%Y-%0m-%0e %H:%M:%S" ) echo -e "${tstamp_H} $0($$): $@" >> ${LOGFILE} } -function ERRLOG () -{ +function ERRLOG { LOG "ERROR: $@" } -function WARNLOG () -{ +function WARNLOG { LOG "WARN: $@" } diff --git a/tox.ini b/tox.ini index c2904c901..a1a548035 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,12 @@ deps = -r{toxinidir}/test-requirements.txt [testenv:linters] whitelist_externals = bash +#bashate ignore +#E006 Line too long +#E041 Arithmetic expansion using $[ is deprecated for $(( +#E042 local declaration hides errors +#E043 Arithmetic compound has inconsistent return semantics +#E044 Use [[ for non-POSIX comparisions commands = bash -c "find {toxinidir} \ -not \( -type d -name .?\* -prune \) \ @@ -21,10 +27,10 @@ commands = -not -name \*~ \ -not -name \*.md \ -name \*.sh \ - -print0 | xargs -0 bashate -v" + -print0 | xargs -0 bashate -v \ + -i E006,E041,E042,E043,E044 -e E*" bash -c "find {toxinidir} \ - \( -name middleware/io-monitor/recipes-common/io-monitor/io-monitor/io_monitor/test-tools/yaml/* -prune \) \ - -o \( -name .tox -prune \) \ + \( -path {toxinidir}/.tox \) -a -prune \ -o -type f -name '*.yaml' \ -print0 | xargs -0 yamllint" diff --git a/utilities/platform-util/scripts/cgcs_tc_setup.sh b/utilities/platform-util/scripts/cgcs_tc_setup.sh index 9507ae978..9dc3542af 100755 --- a/utilities/platform-util/scripts/cgcs_tc_setup.sh +++ b/utilities/platform-util/scripts/cgcs_tc_setup.sh @@ -22,15 +22,13 @@ fi # network link to autonegotiate link speed. Re-run the script in # the background so the parent can return right away and init can # continue. -if [ $# -eq 3 ] -then +if [ $# -eq 3 ]; then $0 $DEV $NETWORKTYPE $NETWORKSPEED dummy & disown exit 0 fi -function test_valid_speed -{ +function test_valid_speed { # After the link is enabled but before the autonegotiation is complete # the link speed may be read as either -1 or as 4294967295 (which is # uint(-1) in twos-complement) depending on the kernel. Neither one is valid. @@ -42,24 +40,20 @@ function test_valid_speed fi } -function log -{ +function log { # It seems that syslog isn't yet running, so append directly to the syslog file echo `date +%FT%T.%3N` `hostname` CGCS_TC_SETUP: $@ >> /var/log/platform.log } -function infra_exists -{ - if [ -z "$infrastructure_interface" ] - then +function infra_exists { + if [ -z "$infrastructure_interface" ]; then return 1 else return 0 fi } -function is_consolidated -{ +function is_consolidated { if ! infra_exists then return 1 @@ -67,26 +61,22 @@ function is_consolidated # determine whether the management interface is a parent of the # infrastructure interface based on name. # eg. this matches enp0s8 to enp0s8.10 but not enp0s88 - if [[ $infrastructure_interface =~ $management_interface[\.][0-9]+$ ]] - then + if [[ $infrastructure_interface =~ $management_interface[\.][0-9]+$ ]]; then return 0 fi return 1 fi } -function is_vlan -{ - if [ -f /proc/net/vlan/$DEV ] - then +function is_vlan { + if [ -f /proc/net/vlan/$DEV ]; then return 0 else return 1 fi } -function is_loopback -{ +function is_loopback { # (from include/uapi/linux/if.h) IFF_LOOPBACK=$((1<<3)) @@ -101,8 +91,7 @@ function is_loopback fi } -function setup_tc_port_filter -{ +function setup_tc_port_filter { local PORT=$1 local PORTMASK=$2 local FLOWID=$3 diff --git a/utilities/platform-util/scripts/remotelogging_tc_setup.sh b/utilities/platform-util/scripts/remotelogging_tc_setup.sh index 4f44fa240..f70da1ec0 100755 --- a/utilities/platform-util/scripts/remotelogging_tc_setup.sh +++ b/utilities/platform-util/scripts/remotelogging_tc_setup.sh @@ -9,8 +9,7 @@ # $1 - listening port of remote log server PORT=$1 -function is_loopback -{ +function is_loopback { # (from include/uapi/linux/if.h) IFF_LOOPBACK=$((1<<3)) @@ -25,14 +24,12 @@ function is_loopback fi } -function log -{ +function log { # It seems that syslog isn't yet running, so append directly to the syslog file echo `date +%FT%T.%3N` `hostname` CGCS_TC_SETUP: $@ >> /var/log/platform.log } -function test_valid_speed -{ +function test_valid_speed { # After the link is enabled but before the autonegotiation is complete # the link speed may be read as either -1 or as 4294967295 (which is # uint(-1) in twos-complement) depending on the kernel. Neither one is valid. @@ -44,8 +41,7 @@ function test_valid_speed fi } -function get_dev_speed -{ +function get_dev_speed { # If the link doesn't come up we won't go enabled, so here we can # afford to wait forever for the link. while true From 5cbb31627aa19324cb71a72a487f1f2a796c41bc Mon Sep 17 00:00:00 2001 From: slin14 Date: Fri, 7 Sep 2018 23:54:21 +0800 Subject: [PATCH 6/9] remove novnc src rpm Story: 2003340 Task: 24378 Depends-On: https://review.openstack.org/600655 It is replaced with rpm package, since no patch for it needed. Change-Id: Ie9cf1fbf3450f85f5ab0fdb34311b8d1799c7d58 Signed-off-by: slin14 --- base/novnc/centos/build_srpm.data | 1 - ...ate-package-0.6.2-versioning-for-TIS-format.patch | 12 ------------ base/novnc/centos/meta_patches/PATCH_ORDER | 1 - base/novnc/centos/srpm_path | 1 - centos_pkg_dirs | 1 - 5 files changed, 16 deletions(-) delete mode 100644 base/novnc/centos/build_srpm.data delete mode 100644 base/novnc/centos/meta_patches/0001-Update-package-0.6.2-versioning-for-TIS-format.patch delete mode 100644 base/novnc/centos/meta_patches/PATCH_ORDER delete mode 100644 base/novnc/centos/srpm_path diff --git a/base/novnc/centos/build_srpm.data b/base/novnc/centos/build_srpm.data deleted file mode 100644 index 8aeb55368..000000000 --- a/base/novnc/centos/build_srpm.data +++ /dev/null @@ -1 +0,0 @@ -TIS_PATCH_VER=1 diff --git a/base/novnc/centos/meta_patches/0001-Update-package-0.6.2-versioning-for-TIS-format.patch b/base/novnc/centos/meta_patches/0001-Update-package-0.6.2-versioning-for-TIS-format.patch deleted file mode 100644 index 062bfe822..000000000 --- a/base/novnc/centos/meta_patches/0001-Update-package-0.6.2-versioning-for-TIS-format.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/SPECS/novnc.spec b/SPECS/novnc.spec -index a43f435..640bf1f 100644 ---- a/SPECS/novnc.spec -+++ b/SPECS/novnc.spec -@@ -1,6 +1,6 @@ - Name: novnc - Version: 0.6.2 --Release: 1%{?dist} -+Release: 1.el7%{?_tis_dist}.%{tis_patch_ver} - Summary: VNC client using HTML5 (Web Sockets, Canvas) with encryption support - Requires: python-websockify - diff --git a/base/novnc/centos/meta_patches/PATCH_ORDER b/base/novnc/centos/meta_patches/PATCH_ORDER deleted file mode 100644 index 0eca01bae..000000000 --- a/base/novnc/centos/meta_patches/PATCH_ORDER +++ /dev/null @@ -1 +0,0 @@ -0001-Update-package-0.6.2-versioning-for-TIS-format.patch diff --git a/base/novnc/centos/srpm_path b/base/novnc/centos/srpm_path deleted file mode 100644 index d59f7598d..000000000 --- a/base/novnc/centos/srpm_path +++ /dev/null @@ -1 +0,0 @@ -mirror:Source/novnc-0.6.2-1.el7.src.rpm diff --git a/centos_pkg_dirs b/centos_pkg_dirs index 7d3e286f1..25c821ae2 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -34,7 +34,6 @@ base/shadow-utils security/shim-unsigned security/shim-signed logging/syslog-ng -base/novnc base/sanlock base/sudo virt/cloud-init From 29a3ab73548c8cad8082f8bba1bed3b60d1851bd Mon Sep 17 00:00:00 2001 From: Jack Ding Date: Thu, 6 Sep 2018 16:01:44 -0400 Subject: [PATCH 7/9] Remove customizations to memcached package Keep memcached package intact and customize memcached service file by overwriting it from platform-util. Story: 2002826 Task: 24548 Depends-On: https://review.openstack.org/600867 Change-Id: Ic18d7efc1ea5548dc6245c7e9658843bd8d557cf Signed-off-by: Jack Ding --- base/memcached/centos/build_srpm.data | 1 - ...te-package-versioning-for-TIS-format.patch | 25 ------- ...002-always-restart-memcached-service.patch | 32 --------- ...-and-comment-out-incompatible-servic.patch | 33 --------- .../memcached/centos/meta_patches/PATCH_ORDER | 3 - ...002-always-restart-memcached-service.patch | 26 ------- ...-and-comment-out-incompatible-servic.patch | 67 ------------------- base/memcached/centos/srpm_path | 1 - centos_iso_image.inc | 1 + centos_pkg_dirs | 1 - .../platform-util/centos/build_srpm.data | 2 +- .../platform-util/centos/platform-util.spec | 11 +++ .../platform-util/scripts/memcached.service | 55 +++++++++++++++ 13 files changed, 68 insertions(+), 190 deletions(-) delete mode 100644 base/memcached/centos/build_srpm.data delete mode 100644 base/memcached/centos/meta_patches/0001-Update-package-versioning-for-TIS-format.patch delete mode 100644 base/memcached/centos/meta_patches/0002-always-restart-memcached-service.patch delete mode 100644 base/memcached/centos/meta_patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch delete mode 100644 base/memcached/centos/meta_patches/PATCH_ORDER delete mode 100644 base/memcached/centos/patches/0002-always-restart-memcached-service.patch delete mode 100644 base/memcached/centos/patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch delete mode 100644 base/memcached/centos/srpm_path create mode 100644 utilities/platform-util/scripts/memcached.service diff --git a/base/memcached/centos/build_srpm.data b/base/memcached/centos/build_srpm.data deleted file mode 100644 index 8aeb55368..000000000 --- a/base/memcached/centos/build_srpm.data +++ /dev/null @@ -1 +0,0 @@ -TIS_PATCH_VER=1 diff --git a/base/memcached/centos/meta_patches/0001-Update-package-versioning-for-TIS-format.patch b/base/memcached/centos/meta_patches/0001-Update-package-versioning-for-TIS-format.patch deleted file mode 100644 index 5625e1b28..000000000 --- a/base/memcached/centos/meta_patches/0001-Update-package-versioning-for-TIS-format.patch +++ /dev/null @@ -1,25 +0,0 @@ -From de355606dea0404c4ae92bad5ce00b841697c698 Mon Sep 17 00:00:00 2001 -From: Jack Ding -Date: Tue, 8 May 2018 14:29:14 -0400 -Subject: [PATCH] Update package versioning for TIS format - ---- - SPECS/memcached.spec | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/SPECS/memcached.spec b/SPECS/memcached.spec -index 6008493..c8575c8 100644 ---- a/SPECS/memcached.spec -+++ b/SPECS/memcached.spec -@@ -4,7 +4,7 @@ - - Name: memcached - Version: 1.4.39 --Release: 1%{?dist} -+Release: 1.el7%{?_tis_dist}.%{tis_patch_ver} - Epoch: 0 - Summary: High Performance, Distributed Memory Object Cache - --- -1.8.3.1 - diff --git a/base/memcached/centos/meta_patches/0002-always-restart-memcached-service.patch b/base/memcached/centos/meta_patches/0002-always-restart-memcached-service.patch deleted file mode 100644 index 89cf9c2ee..000000000 --- a/base/memcached/centos/meta_patches/0002-always-restart-memcached-service.patch +++ /dev/null @@ -1,32 +0,0 @@ -From f321c8a8b800a7c2ca9394d3c76bec72b98c0d77 Mon Sep 17 00:00:00 2001 -From: Jack Ding -Date: Fri, 11 May 2018 15:38:56 -0400 -Subject: [PATCH] always restart memcached service - ---- - SPECS/memcached.spec | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/SPECS/memcached.spec b/SPECS/memcached.spec -index c8575c8..f389035 100644 ---- a/SPECS/memcached.spec -+++ b/SPECS/memcached.spec -@@ -16,6 +16,7 @@ Source1: memcached.sysconfig - - # https://github.com/memcached/memcached/issues/218 - Patch1: 0001-systemd-fix-upstream-provided-service.patch -+Patch2: 0002-always-restart-memcached-service.patch - - BuildRequires: libevent-devel systemd-units - BuildRequires: perl-generators -@@ -44,6 +45,7 @@ access to the memcached binary include files. - %prep - %setup -q - %patch1 -p1 -b .unit -+%patch2 -p1 - - %build - # compile with full RELRO --- -1.8.3.1 - diff --git a/base/memcached/centos/meta_patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch b/base/memcached/centos/meta_patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch deleted file mode 100644 index 93beb0546..000000000 --- a/base/memcached/centos/meta_patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch +++ /dev/null @@ -1,33 +0,0 @@ -From bb6fd3da3ace960eb587e7ff01d5816ea2baaa54 Mon Sep 17 00:00:00 2001 -From: Jack Ding -Date: Sun, 13 May 2018 18:22:15 -0400 -Subject: [PATCH] Add dependencies and comment out incompatible service - parameters - ---- - SPECS/memcached.spec | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/SPECS/memcached.spec b/SPECS/memcached.spec -index f389035..86653a1 100644 ---- a/SPECS/memcached.spec -+++ b/SPECS/memcached.spec -@@ -17,6 +17,7 @@ Source1: memcached.sysconfig - # https://github.com/memcached/memcached/issues/218 - Patch1: 0001-systemd-fix-upstream-provided-service.patch - Patch2: 0002-always-restart-memcached-service.patch -+Patch3: 0003-Add-dependencies-and-comment-out-incompatible-servic.patch - - BuildRequires: libevent-devel systemd-units - BuildRequires: perl-generators -@@ -46,6 +47,7 @@ access to the memcached binary include files. - %setup -q - %patch1 -p1 -b .unit - %patch2 -p1 -+%patch3 -p1 - - %build - # compile with full RELRO --- -1.8.3.1 - diff --git a/base/memcached/centos/meta_patches/PATCH_ORDER b/base/memcached/centos/meta_patches/PATCH_ORDER deleted file mode 100644 index 163c84245..000000000 --- a/base/memcached/centos/meta_patches/PATCH_ORDER +++ /dev/null @@ -1,3 +0,0 @@ -0001-Update-package-versioning-for-TIS-format.patch -0002-always-restart-memcached-service.patch -0003-Add-dependencies-and-comment-out-incompatible-servic.patch diff --git a/base/memcached/centos/patches/0002-always-restart-memcached-service.patch b/base/memcached/centos/patches/0002-always-restart-memcached-service.patch deleted file mode 100644 index 5f899f13e..000000000 --- a/base/memcached/centos/patches/0002-always-restart-memcached-service.patch +++ /dev/null @@ -1,26 +0,0 @@ -From bb7b75184f7037e6d8d844874ae248fce1d06736 Mon Sep 17 00:00:00 2001 -From: Jack Ding -Date: Fri, 11 May 2018 15:24:28 -0400 -Subject: [PATCH] Always restart memcached service - ---- - scripts/memcached.service | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/scripts/memcached.service b/scripts/memcached.service -index 1bb9d33..8e58485 100644 ---- a/scripts/memcached.service -+++ b/scripts/memcached.service -@@ -71,5 +71,9 @@ RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX - # Takes away the ability to create or manage any kind of namespace - RestrictNamespaces=true - -+# WRS -+Restart=always -+RestartSec=0 -+ - [Install] - WantedBy=multi-user.target --- -1.8.3.1 - diff --git a/base/memcached/centos/patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch b/base/memcached/centos/patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch deleted file mode 100644 index a2ef1db58..000000000 --- a/base/memcached/centos/patches/0003-Add-dependencies-and-comment-out-incompatible-servic.patch +++ /dev/null @@ -1,67 +0,0 @@ -From 1d9f43c5ecb20fe0a2a4abe9b94abd0d389edb40 Mon Sep 17 00:00:00 2001 -From: Jack Ding -Date: Mon, 14 May 2018 22:44:32 -0400 -Subject: [PATCH 2/2] Add dependencies and comment out incompatible service - parameters - ---- - scripts/memcached.service | 14 +++++++------- - 1 file changed, 7 insertions(+), 7 deletions(-) - -diff --git a/scripts/memcached.service b/scripts/memcached.service -index 8e58485..021b8b4 100644 ---- a/scripts/memcached.service -+++ b/scripts/memcached.service -@@ -12,7 +12,7 @@ - [Unit] - Description=memcached daemon - Before=httpd.service --After=network.target -+After=network-online.target - - [Service] - EnvironmentFile=/etc/sysconfig/memcached -@@ -46,34 +46,34 @@ LimitNOFILE=16384 - # Explicit module loading will be denied. This allows to turn off module load and unload - # operations on modular kernels. It is recommended to turn this on for most services that - # do not need special file systems or extra kernel modules to work. --ProtectKernelModules=true -+#ProtectKernelModules=true - - # Kernel variables accessible through /proc/sys, /sys, /proc/sysrq-trigger, /proc/latency_stats, - # /proc/acpi, /proc/timer_stats, /proc/fs and /proc/irq will be made read-only to all processes - # of the unit. Usually, tunable kernel variables should only be written at boot-time, with the - # sysctl.d(5) mechanism. Almost no services need to write to these at runtime; it is hence - # recommended to turn this on for most services. --ProtectKernelTunables=true -+#ProtectKernelTunables=true - - # The Linux Control Groups (cgroups(7)) hierarchies accessible through /sys/fs/cgroup will be - # made read-only to all processes of the unit. Except for container managers no services should - # require write access to the control groups hierarchies; it is hence recommended to turn this on - # for most services --ProtectControlGroups=true -+#ProtectControlGroups=true - - # Any attempts to enable realtime scheduling in a process of the unit are refused. --RestrictRealtime=true -+#RestrictRealtime=true - - # Restricts the set of socket address families accessible to the processes of this unit. - # Protects against vulnerabilities such as CVE-2016-8655 - RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX - - # Takes away the ability to create or manage any kind of namespace --RestrictNamespaces=true -+#RestrictNamespaces=true - - # WRS - Restart=always --RestartSec=0 -+RestartSec=10 - - [Install] - WantedBy=multi-user.target --- -1.8.3.1 - diff --git a/base/memcached/centos/srpm_path b/base/memcached/centos/srpm_path deleted file mode 100644 index dc4b6de47..000000000 --- a/base/memcached/centos/srpm_path +++ /dev/null @@ -1 +0,0 @@ -mirror:Source/memcached-1.4.39-1.el7.src.rpm diff --git a/centos_iso_image.inc b/centos_iso_image.inc index 3ebf82576..328facc58 100644 --- a/centos_iso_image.inc +++ b/centos_iso_image.inc @@ -104,6 +104,7 @@ collector # platform-util platform-util platform-util-noncontroller +platform-util-controller # monitor-tools monitor-tools diff --git a/centos_pkg_dirs b/centos_pkg_dirs index 7d3e286f1..bc6e1a196 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -97,7 +97,6 @@ security/tboot networking/mellanox/libibverbs kernel/kernel-modules/mlnx-ofa_kernel networking/mellanox/rdma-core -base/memcached config/puppet-modules/openstack/puppet-memcached-3.0.2 config/puppet-modules/openstack/puppet-horizon-9.5.0 config/puppet-modules/openstack/puppet-swift-11.3.0 diff --git a/utilities/platform-util/centos/build_srpm.data b/utilities/platform-util/centos/build_srpm.data index 8cae09b1e..c2ecd7719 100644 --- a/utilities/platform-util/centos/build_srpm.data +++ b/utilities/platform-util/centos/build_srpm.data @@ -1,4 +1,4 @@ SRC_DIR="platform-util" COPY_LIST_TO_TAR="scripts" -TIS_PATCH_VER=12 +TIS_PATCH_VER=13 diff --git a/utilities/platform-util/centos/platform-util.spec b/utilities/platform-util/centos/platform-util.spec index 12d9c510f..946b33da5 100644 --- a/utilities/platform-util/centos/platform-util.spec +++ b/utilities/platform-util/centos/platform-util.spec @@ -20,6 +20,12 @@ Summary: non controller platform utilities %description -n platform-util-noncontroller Platform utilities that don't get packaged on controller hosts +%package -n platform-util-controller +Summary: controller platform utilities + +%description -n platform-util-controller +Platform utilities that packaged on controllers or one node system + %define local_dir /usr/local %define local_bindir %{local_dir}/bin %define local_sbindir %{local_dir}/sbin @@ -52,6 +58,7 @@ install -m 700 -p -D %{_buildsubdir}/scripts/patch-restart-haproxy %{buildroot}% install -d %{buildroot}/etc/systemd/system install -m 644 -p -D %{_buildsubdir}/scripts/opt-platform.mount %{buildroot}/etc/systemd/system install -m 644 -p -D %{_buildsubdir}/scripts/opt-platform.service %{buildroot}/etc/systemd/system +install -m 644 -p -D %{_buildsubdir}/scripts/memcached.service %{buildroot}/etc/systemd/system # Mask the systemd ctrl-alt-delete.target, to disable reboot on ctrl-alt-del ln -sf /dev/null %{buildroot}/etc/systemd/system/ctrl-alt-del.target @@ -85,3 +92,7 @@ systemctl enable opt-platform.service # from parsing the fstab is not used by systemd. /etc/systemd/system/opt-platform.mount /etc/systemd/system/opt-platform.service + +%files -n platform-util-controller +%defattr(-,root,root,-) +/etc/systemd/system/memcached.service diff --git a/utilities/platform-util/scripts/memcached.service b/utilities/platform-util/scripts/memcached.service new file mode 100644 index 000000000..8dd1e891c --- /dev/null +++ b/utilities/platform-util/scripts/memcached.service @@ -0,0 +1,55 @@ +# +# This service file is a customized version in platform-util package from +# openstack/stx-integ project + +[Unit] +Description=memcached daemon +Before=httpd.service +After=network-online.target + +[Service] +EnvironmentFile=/etc/sysconfig/memcached +ExecStart=/usr/bin/memcached -p ${PORT} -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS + +# Set up a new file system namespace and mounts private /tmp and /var/tmp directories +# so this service cannot access the global directories and other processes cannot +# access this service's directories. +PrivateTmp=true + +# Mounts the /usr, /boot, and /etc directories read-only for processes invoked by this unit. +ProtectSystem=full + +# Ensures that the service process and all its children can never gain new privileges +NoNewPrivileges=true + +# Sets up a new /dev namespace for the executed processes and only adds API pseudo devices +# such as /dev/null, /dev/zero or /dev/random (as well as the pseudo TTY subsystem) to it, +# but no physical devices such as /dev/sda. +PrivateDevices=true + +# Required for dropping privileges and running as a different user +CapabilityBoundingSet=CAP_SETGID CAP_SETUID CAP_SYS_RESOURCE +LimitNOFILE=16384 + +# Attempts to create memory mappings that are writable and executable at the same time, +# or to change existing memory mappings to become executable are prohibited. +# XXX: this property is supported with systemd 231+ which is not yet on EL7 +# MemoryDenyWriteExecute=true + +# Restricts the set of socket address families accessible to the processes of this unit. +# Protects against vulnerabilities such as CVE-2016-8655 +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX + +# These service parameters are commented out since they are incompatible with +# Centos 7 and generate warning messages when included. +#ProtectKernelModules=true +#ProtectKernelTunables=true +#ProtectControlGroups=true +#RestrictRealtime=true +#RestrictNamespaces=true + +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target From c93b246e75d5bd6038724a52df9ab8532fb2ada5 Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Mon, 10 Sep 2018 14:23:39 -0400 Subject: [PATCH 8/9] Exclude /etc files that are causing collect extraction errors The following files are causing collect log extraction errors. /etc/systemd/system/lvm2-lvmetad.socket /etc/systemd/system/lvm2-lvmetad.service This update excludes all socket files in /etc/systemd/system and the /etc/systemd/system/lvm2-lvmetad.service from being collected as part of the collect operation. In doing so the extraction error does not occur. Note: Don't want to exclude all service files. Change-Id: Iaa34fd9b2dbd072980c1487278fb0e2f4cb1ec14 Related-Bug: https://bugs.launchpad.net/starlingx/+bug/1791780 Signed-off-by: Eric MacDonald --- tools/collector/centos/build_srpm.data | 2 +- tools/collector/scripts/etc.exclude | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/collector/centos/build_srpm.data b/tools/collector/centos/build_srpm.data index 69bf0121c..66f0fdbd8 100644 --- a/tools/collector/centos/build_srpm.data +++ b/tools/collector/centos/build_srpm.data @@ -1,2 +1,2 @@ SRC_DIR="scripts" -TIS_PATCH_VER=24 +TIS_PATCH_VER=25 diff --git a/tools/collector/scripts/etc.exclude b/tools/collector/scripts/etc.exclude index d3f3827d8..d870e9b22 100644 --- a/tools/collector/scripts/etc.exclude +++ b/tools/collector/scripts/etc.exclude @@ -35,6 +35,8 @@ /etc/bash_completion.d /etc/pm /etc/systemd/system/*.mount +/etc/systemd/system/*.socket +/etc/systemd/system/lvm2-lvmetad.service /etc/systemd/system/ctrl-alt-del.target /etc/ssl /etc/mtc/tmp From 3cd5c19b006f8f07abd9dc4d8cfa4acb926cc401 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Thu, 6 Sep 2018 10:28:09 -0400 Subject: [PATCH 9/9] Fix typo: strorage-drivers to storage-drivers Closes-Bug: 1791106 Change-Id: Ie6be8ce67e30ed49c3007a819637ae2bd332046f Signed-off-by: Scott Little --- centos_pkg_dirs | 4 ++-- .../python-3parclient/centos/build_srpm.data | 0 .../python-3parclient/centos/python-3parclient.spec | 0 .../python-lefthandclient/centos/build_srpm.data | 0 .../python-lefthandclient/centos/python-lefthandclient.spec | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename {strorage-drivers => storage-drivers}/python-3parclient/centos/build_srpm.data (100%) rename {strorage-drivers => storage-drivers}/python-3parclient/centos/python-3parclient.spec (100%) rename {strorage-drivers => storage-drivers}/python-lefthandclient/centos/build_srpm.data (100%) rename {strorage-drivers => storage-drivers}/python-lefthandclient/centos/python-lefthandclient.spec (100%) diff --git a/centos_pkg_dirs b/centos_pkg_dirs index 47110fed8..927a32b22 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -89,8 +89,8 @@ python/python-requests base/systemd python/python-gunicorn config/puppet-modules/openstack/puppet-oslo-11.3.0 -strorage-drivers/python-3parclient -strorage-drivers/python-lefthandclient +storage-drivers/python-3parclient +storage-drivers/python-lefthandclient networking/iproute security/tboot networking/mellanox/libibverbs diff --git a/strorage-drivers/python-3parclient/centos/build_srpm.data b/storage-drivers/python-3parclient/centos/build_srpm.data similarity index 100% rename from strorage-drivers/python-3parclient/centos/build_srpm.data rename to storage-drivers/python-3parclient/centos/build_srpm.data diff --git a/strorage-drivers/python-3parclient/centos/python-3parclient.spec b/storage-drivers/python-3parclient/centos/python-3parclient.spec similarity index 100% rename from strorage-drivers/python-3parclient/centos/python-3parclient.spec rename to storage-drivers/python-3parclient/centos/python-3parclient.spec diff --git a/strorage-drivers/python-lefthandclient/centos/build_srpm.data b/storage-drivers/python-lefthandclient/centos/build_srpm.data similarity index 100% rename from strorage-drivers/python-lefthandclient/centos/build_srpm.data rename to storage-drivers/python-lefthandclient/centos/build_srpm.data diff --git a/strorage-drivers/python-lefthandclient/centos/python-lefthandclient.spec b/storage-drivers/python-lefthandclient/centos/python-lefthandclient.spec similarity index 100% rename from strorage-drivers/python-lefthandclient/centos/python-lefthandclient.spec rename to storage-drivers/python-lefthandclient/centos/python-lefthandclient.spec