#!/bin/bash

#Copyright (c) 2016-2017 Wind River Systems, Inc.
#
#SPDX-License-Identifier: Apache-2.0
#

# This script is used to parse all stats data. It is designed to be called by either
# parse-controllers.sh or parse-computes.sh and not used as a standalone script.
# If the input node is a controller, it will parse controller specific postgres &
# and rabbitmq stats first. If the input node is a compute, it will pars the compute 
# specific vswitch stats first.
#
# The following parsing steps are common to all hosts and are executed in the specified order:
#     - Parse occtop
#     - Parse memtop
#     - Parse memstats (summary)
#     - Parse netstats
#     - Parse schedtop (summary)
#     - Parse iostats
#     - Parse diskstats
#     - Parse filestats (summary)
#     - Parse process level schedtop (optional step, configured in lab.conf)
#     - Generate tarball
 
if [[ $# != 1 ]]; then
   echo "ERROR: This script is meant to be called by either parse-controllers.sh or parse-computes.sh script."
   echo "To run it separately, copy the script to the host directory that contains *.bz2 files."
   echo "It takes a single argument - the name of the host directory (e.g. ./parse-all.sh controller-0)."
   exit 1
fi

source ../lab.conf
source ./host.conf

PARSERDIR=$(dirname $0)
. ${PARSERDIR}/parse-util.sh

NODE=$1

CURDATE=$(date)
DATESTAMP=$(date +%b-%d)

function sedit()
{
    local FILETOSED=$1
    sed -i -e "s/  */ /g" ${FILETOSED}
    sed -i -e "s/ /,/g" ${FILETOSED}
    # Remove any trailing comma
    sed -i "s/,$//" ${FILETOSED}
}
    
function get_filename_from_mountname()
{
    local name=$1
    local fname
    if test "${name#*"scratch"}" != "${name}"; then
        fname="scratch"
    elif test "${name#*"log"}" != "${name}"; then
        fname="log"
    elif test "${name#*"backup"}" != "${name}"; then
        fname="backup"
    elif test "${name#*"ceph/mon"}" != "${name}"; then
        fname="cephmon"
    elif test "${name#*"conversion"}" != "${name}"; then
        fname="img-conversion"
    elif test "${name#*"platform"}" != "${name}"; then
        fname="platform"
    elif test "${name#*"postgres"}" != "${name}"; then
        fname="postgres"
    elif test "${name#*"cgcs"}" != "${name}"; then
        fname="cgcs"
    elif test "${name#*"rabbitmq"}" != "${name}"; then
        fname="rabbitmq"
    elif test "${name#*"instances"}" != "${name}"; then
        fname="pv"
    elif test "${name#*"ceph/osd"}" != "${name}"; then
        # The ceph disk partition has the following mount name convention
        # /var/lib/ceph/osd/ceph-0
        fname=`basename ${name}`
    fi
    echo $fname
}

function parse_process_schedtop_data()
{
    # Logic has been moved to a separate script so that parsing process level schedtop
    # can be run either as part of parse-all.sh script or independently.
    LOG "Process level schedtop parsing is turned on in lab.conf. Parsing schedtop detail..."
    cd ..
    ./parse-schedtop.sh ${NODE}
    cd ${NODE}
}

function parse_controller_specific()
{
    # Parsing Postgres data, removing data from previous run if there are any. Generate summary
    # data for each database and detail data for specified tables
    LOG "Parsing postgres data for ${NODE}"
    if [ -z "${DATABASE_LIST}" ]; then
        WARNLOG "DATABASE_LIST is not set in the lab.conf file. Use default setting"
        DATABASE_LIST="cinder glance keystone nova neutron ceilometer heat sysinv aodh postgres nova_api"
    fi

    for DB in ${DATABASE_LIST}; do
        rm /tmp/${DB}*.csv
    done
    ../parse_postgres *postgres.bz2 >postgres-summary-${NODE}-${DATESTAMP}.txt
    for DB in ${DATABASE_LIST}; do
        cp /tmp/${DB}_size.csv postgres_${DB}_size.csv
    done
    for TABLE in ${TABLE_LIST}; do
        cp /tmp/${TABLE}.csv postgres_${TABLE}.csv
    done

    # Parsing RabbitMQ data
    LOG "Parsing rabbitmq data for ${NODE}"
    ../parse-rabbitmq.sh rabbitmq-${NODE}.csv

    for QUEUE in ${RABBITMQ_QUEUE_LIST}; do
        # If node is not a controller node then parse-rabbitmq-queue.sh should skip
        ../parse-rabbitmq-queue.sh rabbitmq-${QUEUE}-${NODE}.csv ${QUEUE}
    done
}

function parse_compute_specific()
{
    LOG "Parsing vswitch data for ${NODE}"
    ../parse-vswitch.sh ${NODE}
}

function parse_occtop_data()
{
    LOG "Parsing occtop data for ${NODE}"
    bzcat *occtop.bz2 >occtop-${NODE}-${DATESTAMP}.txt
    cp occtop-${NODE}-${DATESTAMP}.txt tmp.txt
    sedit tmp.txt
    # Get the highest column count
    column_count=$(awk -F "," '{print NF}' tmp.txt | sort -nu | tail -n 1)
    grep '^[0-9]' tmp.txt |cut -d, -f1,2 | awk -F "," '{print $1" "$2}' > tmpdate.txt
    grep '^[0-9]' tmp.txt |cut -d, -f3-$column_count > tmpcore.txt
    paste -d, tmpdate.txt tmpcore.txt > tmp2.txt
    # Generate header based on the number of columns. The Date/Time column consists of date and time fields
    header="Date/Time,Total"
    count=$(($column_count-3))
    for i in $(seq 0 $(($count-1))); do
        header="$header,$i"
    done

    # Generate detailed CSV with Date/Time, Total CPU occupancy and individual core occupancies e.g.
    # Date/Time,Total,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35
    # 2016-11-22 00:29:16.523,759.5,21.4,18.9,43.8,24.5,23.1,25.3,28.1,25.5,20.5,27.8,26.8,32.7,27.3,25.1,21.1,23.2,21.7,36.4,23.3,16.6,15.3,13.9,14.4,15.0,14.7,14.4,16.4,13.8,17.0,17.8,19.0,15.1,14.0,13.2,14.5,17.8
    echo "${header}" > occtop-${NODE}-detailed.csv
    cat tmp2.txt >> occtop-${NODE}-detailed.csv

    # Generate simple CSV file which is used to generate host CPU occupancy chart. Platform cores are 
    # defined in the host.conf. The simple CSV contains only the Date/Time and Total platform CPU occupancy e.g.
    # Date/Time,Total
    # 2016-11-22 00:29:16.523,94.9
    # 2016-11-22 00:30:16.526,71.3

    if [ -z "${PLATFORM_CPU_LIST}" ]; then
        # A controller node in standard system. In this case, all cores are dedicated to platform use.
        # Simply extract the Date/Time and Total CPU occupancy
        cut -d, -f1,2 occtop-${NODE}-detailed.csv > occtop-${NODE}.csv
    else
        # A CPE, compute or storage node. The cores dedicated to platform use are specified in the config.
        echo "Date/Time,Total" > occtop-${NODE}.csv
        while read -r line || [[ -n "$line" ]]; do
            IFS="," read -r -a arr <<< "${line}"
            total=0
            for CORE in ${PLATFORM_CPU_LIST}; do
                # Add 2 to the index as occupancy of each individual core starts after Date/Time and Total
                idx=$(($CORE+2))
                total=`echo $total + ${arr[$idx]} | bc`
            done
            echo "${arr[0]},${total}" >> occtop-${NODE}.csv
        done < tmp2.txt
    fi
    # Remove temporary files
    rm tmp.txt tmp2.txt tmpdate.txt tmpcore.txt
}

function parse_memtop_data()
{
    LOG "Parsing memtop data for ${NODE}"
    bzcat *memtop.bz2 > memtop-${NODE}-${DATESTAMP}.txt
    cp memtop-${NODE}-${DATESTAMP}.txt tmp.txt
    sedit tmp.txt
  
    # After dumping all memtop bz2 output into one text file and in-place sed, grab only relevant data
    # for CSV output. Generate both detailed and simple CSV files. Simple output will be used to generate
    # chart.
    grep '^[0-9]' tmp.txt | awk -F "," '{print $1" "$2","$3","$4","$5","$6","$7","$8","$9","$10","$11","$12","$13","$14","$15","$16","$17","$18}' > tmp2.txt
    echo "Date/Time,Total,Used,Free,Cached,Buf,Slab,CAS,CLim,Dirty,WBack,Anon,Avail,0:Avail,0:HFree,1:Avail,1:HFree" > memtop-${NODE}-detailed.csv
    cat tmp2.txt >> memtop-${NODE}-detailed.csv
    echo "Date/Time,Total,Anon" > memtop-${NODE}.csv
    cut -d, -f1-2,12 tmp2.txt >> memtop-${NODE}.csv
    # Remove temporary files
    rm tmp.txt tmp2.txt
}

function parse_netstats_data()
{
    LOG "Parsing netstats data for ${NODE}"
    # First generate the summary data then detail data for specified interfaces
    ../parse_netstats *netstats.bz2 > netstats-summary-${NODE}-${DATESTAMP}.txt
    if [ -z "${NETSTATS_INTERFACE_LIST}" ]; then
        ERRLOG "NETSTATS_INTERFACE_LIST is not set in host.conf. Skipping detail netstats..."
    else
        for INTERFACE in ${NETSTATS_INTERFACE_LIST}; do
            echo "Date/Time,Interface,Rx PPS,Rx Mbps,Rx Packet Size,Tx PPS,Tx Mbps,Tx Packet Size" > netstats-${NODE}-${INTERFACE}.csv
            ../parse_netstats *netstats.bz2 | grep " ${INTERFACE} " > tmp.txt
            sed -i -e "s/|/ /g" tmp.txt
            sed -i -e "s/    */ /g;s/  */ /g" tmp.txt
            sed -i -e "s/ /,/g" tmp.txt
            # Remove the leading comma
            sed -i 's/,//' tmp.txt
            while read -r line || [[ -n "$line" ]]; do
                IFS="," read -r -a arr <<< "${line}"
                echo "${arr[8]} ${arr[9]},${arr[0]},${arr[2]},${arr[3]},${arr[4]},${arr[5]},${arr[6]},${arr[7]}" >> netstats-${NODE}-${INTERFACE}.csv
            done < tmp.txt
        done
        rm tmp.txt
    fi 
}

function parse_iostats_data()
{
    LOG "Parsing iostat data for ${NODE}"
    if [ -z "${IOSTATS_DEVICE_LIST}" ]; then
        ERRLOG "IOSTAT_DEVICE_LIST is not set in host.conf. Skipping iostats..."
    else
        for DEVICE in ${IOSTATS_DEVICE_LIST}; do
            # Add header to output csv file
            echo "Date/Time,${DEVICE},rqm/s,wrqm/s,r/s,w/s,rkB/s,wkB/s,avgrq-sz,avgqu-sz,await,r_await,w_await,svctm,%util" > iostat-${NODE}-${DEVICE}.csv
            # Dumping iostat content to tmp file
            bzcat *iostat.bz2 | grep -E "/2015|/2016|/2017|${DEVICE}"   | awk '{print $1","$2","$3","$4","$5","$6","$7","$8","$9","$10","$11","$12","$13","$14}' > tmp.txt
            while IFS= read -r current
            do
               if test "${current#*Linux}" != "$current"
               then
		   # Skip the line that contains the word "Linux"
                   continue
               else
                   if test "${current#*$DEVICE}" == "$current"
                   then
                       # It's a date entry, look ahead
                       read -r next
                       if test "${next#*$DEVICE}" != "${next}"
                       then
			   # This next line contains the device stats
			   # Combine date and time fields
                           current="${current//2016,/2016 }"
                           current="${current//2017,/2017 }"
			   # Combine time and AM/PM fields
                           current="${current//,AM/ AM}"
                           current="${current//,PM/ PM}"
                           # Write both lines to intermediate file
                           echo "${current}" >> tmp2.txt
                           echo "${next}" >> tmp2.txt
                       fi
                    fi
                fi
            done < tmp.txt
            mv tmp2.txt tmp.txt
            # Combine the time and stats data into one line
            # 11/22/2016 06:34:00 AM,,,,,,,,,,,
            # dm-0,0.00,0.00,0.00,1.07,0.00,38.00,71.25,0.00,0.19,0.00,0.19,0.03,0.00
            paste -d "" - - < tmp.txt > tmp2.txt
            # Remove empty fields, use "complement" option for contiguous fields/range
            cut -d, -f2-11 --complement tmp2.txt > tmp.txt
            # Write final content to output csv
            cat tmp.txt >> iostat-${NODE}-${DEVICE}.csv
	    rm tmp.txt tmp2.txt
        done
    fi
}

function parse_diskstats_data()
{
    LOG "Parsing diskstats data for ${NODE}"

    if [ -z "${DISKSTATS_FILESYSTEM_LIST}" ]; then
        ERRLOG "DISKSTATS_FILESYSTEM_LIST is not set in host.conf. Skipping diskstats..."
    else
        for FS in ${DISKSTATS_FILESYSTEM_LIST}; do
            fspair=(${FS//|/ })
            fsname=${fspair[0]}
            mountname=${fspair[1]}
            if [ ${mountname} == "/" ]; then
                mountname=" /"
                echo "File system,Type,Size,Used,Avail,Used(%)" > diskstats-${NODE}-root.csv
                bzcat *diskstats.bz2 | grep $fsname | grep $mountname | grep G | awk '{print $1","$2","$3","$4","$5","$6}' >> diskstats-${NODE}-root.csv
            else
                fname=$(get_filename_from_mountname $mountname)
                echo "File system,Type,Size,Used,Avail,Used(%)" > diskstats-${NODE}-$fname.csv
                bzcat *diskstats.bz2 | grep $fsname | grep $mountname | grep G | awk '{print $1","$2","$3","$4","$5","$6}' >> diskstats-${NODE}-$fname.csv
            fi
        done
    fi
}

# Parsing starts here ...
LOG "Parsing ${NODE} files - ${CURDATE}"

# Let's get the host specific parsing out of the way
if test "${NODE#*"controller"}" != "${NODE}"; then
    parse_controller_specific
elif test "${NODE#*"compute"}" != "${NODE}"; then
    parse_compute_specific
fi

# Parsing CPU occtop data
parse_occtop_data

# Parsing memtop data
parse_memtop_data

# Parsing memstats data to generate the high level report. The most important piece of info is the list of 
# hi-runners at the end of the file. If there is a leak, run parse-daily.sh script to generate the time
# series data for the offending processes only. Use process name, not PID as most Titanium Cloud processes have 
# workers.
LOG "Parsing memstats summary for ${NODE}"
../parse_memstats --report *memstats.bz2 > memstats-summary-${NODE}-${DATESTAMP}.txt
#tar czf pidstats.tgz pid-*.csv
rm pid-*.csv


# Parsing netstats data
parse_netstats_data

# Parsing schedtop data to generate the high level report. Leave the process level schedtop parsing till
# the end as it is a long running task.   
LOG "Parsing schedtop summary for ${NODE}"
FILES=$(ls *schedtop.bz2)
../parse_schedtop ${FILES} > schedtop-summary-${NODE}-${DATESTAMP}.txt

# Parsing iostats data
parse_iostats_data

# Parsing diskstats data
parse_diskstats_data

# Parsing filestats data to generate the high level report. If there is a file descriptor leak, run parse-daily.sh 
# script to generate the time series data for the offending processes only. Use process name, not PID as most 
# Titanium Cloud processes have workers.
LOG "Parsing filestats summary for ${NODE}"
../parse_filestats --all *filestats.bz2 > filestats-summary-${NODE}-${DATESTAMP}.txt 

# Parsing process level schedtop data. This is a long running task. To skip this step or generate data for
# only specific processes, update the lab.conf and host.conf files.
[[ ${GENERATE_PROCESS_SCHEDTOP} == Y ]] && parse_process_schedtop_data || WARNLOG "Parsing process level schedtop is skipped."

# Done parsing for this host. If it's a controller host, check if the parsing of postgres connection stats which is run in 
# parallel is done before creating a tar file.
if test "${NODE#*"controller"}" != "${NODE}"; then
    # If postgres-conns.csv file has not been created which is highly unlikely, wait a couple of minutes
    [ ! -e postgres-conns.csv ] && sleep 120

    # Now check the stats of this file every 5 seconds to see if it's still being updated. Another option
    # is to use inotify which requires another inotify-tools package.
    oldsize=0
    newsize=0
    while true
    do
        newsize=$(stat -c %s postgres-conns.csv)
        if [ "$oldsize" == "$newsize" ]; then
            break
        fi
        oldsize=$newsize
        sleep 5
    done
fi
tar czf syseng-data-${NODE}-${DATESTAMP}.tgz *.csv *.txt
LOG "Parsing stats data for ${NODE} completed!"