#!/bin/bash
#
# Copyright (c) 2016 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
##############################################################################
#
# This script supports no-reboot patching of any single or
# combination of maintenance processes specified on the command line.
#
# Calling sequence:
#
# rc=mtce-restart process1 process2 process3 ...
# if [ $? != 0 ] ; then
#    restart action failed
#
#
###############################################################################
#
# The patching subsystem provides a patch-functions bash source file
# with useful function and variable definitions.
#
if [ -e "/etc/patching/patch-functions" ] ; then
   . /etc/patching/patch-functions
fi

loginfo "----------------------------------------------"
loginfo "Maintenance No-Reboot Patching Restart Request"

#
# Declare an overall script return code
#
declare -i GLOBAL_RC=$PATCH_STATUS_FAILED

#if [ ! -e $PATCH_FLAGDIR ] ; then
#    mkdir -p $PATCH_FLAGDIR
#fi

# if set with -c or --clean options then the flag files for
# each process are removed at the start.
CLEAN=false

#
# Completion status ; stored in PID index
#
DISABLED="disabled"
NOPID="not-running"
SKIPPED="skipped"
RESTARTED="restarted"

#
# process query and restart executables
#
SM_RESTART_EXEC="sm-restart-safe"
SM_QUERY_EXEC="sm-query"
PMON_RESTART_EXEC="pmon-restart"

#
# Struct indexes
#
PROCESS_INDEX=0
PID_INDEX=1
ALIAS_INDEX=2


#
# Process Struct and List       [ name ] [ alias ] [ pid | status ]
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# NOTE TO PATCH WRITERS: Simply Un-Comment processes you want no-reboot patch restarted.
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#

# The process restart control structure
declare sm_managed_processes=""
declare pmon_managed_processes=""

# Build the process list.
# All arguements should be a valid maintenance process name.
# The name of the binary, not the SM alias.
# See the list below for supported process names.
while [[ ${#} > 0 ]]
do
    process="${1}"
    case $process in

       -c|--clean)
           CLEAN=true
           ;;

       # Maintenance Processes - SM managed
       "mtcAgent")
           sm_managed_processes=(  ${sm_managed_processes[@]} "mtcAgent:0:mtc-agent")
           ;;
       "guestAgent")
           sm_managed_processes=(  ${sm_managed_processes[@]} "guestAgent:0:guest-agent")
           ;;
       "hwmond")
           sm_managed_processes=(  ${sm_managed_processes[@]} "hwmond:0:hw-mon")
           ;;

       # Maintenance Processes - PMON managed
       "pmond")
           pmon_managed_processes=(${pmon_managed_processes[@]} "pmond:0")
           ;;
       "guestServer")
           pmon_managed_processes=(${pmon_managed_processes[@]} "guestServer:0")
           ;;
       "hbsAgent")
           pmon_managed_processes=(${pmon_managed_processes[@]} "hbsAgent:0")
           ;;
       "mtcClient")
           pmon_managed_processes=(${pmon_managed_processes[@]} "mtcClient:0")
           ;;
       "hbsClient")
           pmon_managed_processes=(${pmon_managed_processes[@]} "hbsClient:0")
           ;;
       "rmond")
           pmon_managed_processes=(${pmon_managed_processes[@]} "rmond:0")
           ;;
       "hostwd")
           pmon_managed_processes=(${pmon_managed_processes[@]} "hostwd:0")
           ;;
       "fsmond")
           pmon_managed_processes=(${pmon_managed_processes[@]} "fsmond:0")
           ;;
       "mtclogd")
           pmon_managed_processes=(${pmon_managed_processes[@]} "mtclogd:0")
           ;;
       "mtcalarmd")
           pmon_managed_processes=(${pmon_managed_processes[@]} "mtcalarmd:0")
           ;;
       "lmond")
           pmon_managed_processes=(${pmon_managed_processes[@]} "lmond:0")
           ;;

       *)
           loginfo "Unknown process:${process}"
           ;;
   esac
   shift
done

# Assume both groupings are done until we know there are not
sm_done=true
pmon_done=true

#if [ ${#sm_managed_processes[@]} -ne 0 -a is_controller ] ; then
if [ -n "${sm_managed_processes}" -a is_controller ] ; then

   # Record current process IDs
   index=0
   for DAEMON in "${sm_managed_processes[@]}"
   do
      info=(${DAEMON//:/ })

      if [ "${CLEAN}" = true ] ; then
         rm -f $PATCH_FLAGDIR/${info[${PROCESS_INDEX}]}.restarted
      fi

      info[${PID_INDEX}]=`pidof ${info[${PROCESS_INDEX}]}`
      if [ -z "${info[${PID_INDEX}]}" ] ; then
            loginfo "${info[${PROCESS_INDEX}]} is not running"
            info[${PID_INDEX}]="${NOPID}"
      fi

      # Save the PID or NOPID status to the process line
      sm_managed_processes[${index}]="${info[${PROCESS_INDEX}]}:${info[${PID_INDEX}]}:${info[${ALIAS_INDEX}]}"

      ((index++))
   done

   # Restart the processes
   index=0
   for DAEMON in "${sm_managed_processes[@]}"
   do
      info=(${DAEMON//:/ })

      if [ -e $PATCH_FLAGDIR/${info[${PROCESS_INDEX}]}.restarted ] ; then
          info[${PID_INDEX}]="${SKIPPED}"

          # Add the PID to the process line
          sm_managed_processes[${index}]="${info[${PROCESS_INDEX}]}:${info[${PID_INDEX}]}:${info[${ALIAS_INDEX}]}"
          ((index++))

          continue
      fi
      sm_query_result=`${SM_QUERY_EXEC} service ${info[${ALIAS_INDEX}]}`
      if [[ "${sm_query_result}" == *"enabled-active"* ]] ; then
         # Save the original PID
         info[${PID_INDEX}]=`pidof ${info[${PROCESS_INDEX}]}`

         if [ -n "${info[${PID_INDEX}]}" ] ; then

            loginfo "sm-restart of ${info[${PROCESS_INDEX}]} [pid:${info[${PID_INDEX}]}]"
            touch $PATCH_FLAGDIR/${info[${PROCESS_INDEX}]}.restarted
            ${SM_RESTART_EXEC} service "${info[${ALIAS_INDEX}]}"
            sm_done=false
            sleep 5

         else
            loginfo "${info[${PROCESS_INDEX}]} is not running ; must be on inactive controller"
            info[${PID_INDEX}]="${NOPID}"
         fi
      elif [[ ${sm_query_result} == *"is enabling"* ]] ; then
         info[${PID_INDEX}]="${NOPID}"
         loginfo "sm-restart ${info[${PROCESS_INDEX}]} ; [in progress] ; [pid:${info[${PID_INDEX}]}]"
      else
         info[${PID_INDEX}]="${DISABLED}"
         loginfo "${info[${PROCESS_INDEX}]} is not active"
      fi

      # Add the PID to the process line
      sm_managed_processes[${index}]="${info[${PROCESS_INDEX}]}:${info[${PID_INDEX}]}:${info[${ALIAS_INDEX}]}"

      ((index++))
   done
fi

if [ -n "${pmon_managed_processes}" ] ; then

   echo "DEBUG: pmon_managed_processes:${pmon_managed_processes}"

   # Restart the pmond processes
   index=0
   for DAEMON in "${pmon_managed_processes[@]}"
   do
      info=(${DAEMON//:/ })

      if [ "${CLEAN}" = true ] ; then
         rm -f $PATCH_FLAGDIR/${info[${PROCESS_INDEX}]}.restarted
      fi

      if [ -e $PATCH_FLAGDIR/${info[${PROCESS_INDEX}]}.restarted ] ; then
         info[${PID_INDEX}]="${SKIPPED}"
         pmon_managed_processes[${index}]="${info[${PROCESS_INDEX}]}:${info[${PID_INDEX}]}"
         ((index++))
         continue
      fi

      # Save the original PID
      info[${PID_INDEX}]=`pidof ${info[${PROCESS_INDEX}]}`

      if [ -n "${info[${PID_INDEX}]}" ] ; then
        loginfo "pmon-restart of ${info[${PROCESS_INDEX}]} [pid:${info[${PID_INDEX}]}]"
        touch $PATCH_FLAGDIR/${info[${PROCESS_INDEX}]}.restarted
        ${PMON_RESTART_EXEC} ${info[${PROCESS_INDEX}]}
        pmon_done=false
        sleep 2

        ####################################################################
        # Special Handling Section
        #
        # - pmond needs 30 seconds to restart before it will start
        #   monitoring processes.We can maybe remove that in the daemon
        #   config file but for now its there and we have to wait.
        ####################################################################
        if [ "${info[${PROCESS_INDEX}]}" == "pmond" ] ; then
            sleep 30
        fi

      else
         info[${PID_INDEX}]="${DISABLED}"
         loginfo "${info[${PROCESS_INDEX}]} is not active"
      fi

      # Save the updated PID or other status to the process line
      pmon_managed_processes[${index}]="${info[${PROCESS_INDEX}]}:${info[${PID_INDEX}]}"

      ((index++))
   done
fi

# check for done. If this is not met in timeout then fail is returned
if [ "$sm_done" = true -a "$pmon_done" = true ] ; then
   GLOBAL_RC=$PATCH_STATUS_OK
   loginfo "  SM Processes: ${sm_managed_processes[@]}"
   loginfo "PMON Processes: ${pmon_managed_processes[@]}"
   loginfo "Maintenance No-Reboot Patching Status: ${GLOBAL_RC} - nothing to do."
   exit ${GLOBAL_RC}
fi

# Monitor the restart of SM processes
#
# Don't want to start from the beginning of the shell
# Want time zero now plus 20 seconds.
#
SECONDS=0
TIMEOUT=120
let UNTIL=${SECONDS}+${TIMEOUT}
loginfo "restart timeout is ${TIMEOUT}"

while [ ${UNTIL} -ge ${SECONDS} ]
do
   if [ "$sm_done" = false ] ; then
      if [ is_controller -o is_cpe ] ; then
         sm_not_done=false
         index=0
         for DAEMON in "${sm_managed_processes[@]}"
         do
            info=(${DAEMON//:/ })

            # Don't wast time on processes that are being skipped due to past restart
            if [ "${info[${PID_INDEX}]}" == "${SKIPPED}" ] ; then
               ((index++))
                continue

            # Don't wast time on processes that have already restarted
            elif [ "${info[${PID_INDEX}]}" == "${RESTARTED}" ] ; then
               ((index++))
                continue

            # Don't look for disabled processes
            elif [ "${info[${PID_INDEX}]}" == "${DISABLED}" ] ; then
               ((index++))
                continue

            # Don't look at not running processes
            elif [ "${info[${PID_INDEX}]}" == "${NOPID}" ] ; then
               ((index++))
                continue

            elif [[ `sm-query service ${info[${ALIAS_INDEX}]}` == *"enabled-active"* ]] ; then

               # Save the original PID
               new_pid=`pidof ${info[${PROCESS_INDEX}]}`
               if [ $? -eq 0 -a -n ${new_pid} ] ; then

                  if [ "${info[${PID_INDEX}]}" != "${new_pid}" ] ; then
                     loginfo "${info[${PROCESS_INDEX}]} ${RESTARTED} ok [pid:${info[${PID_INDEX}]} -> ${new_pid}]"
                     info[${PID_INDEX}]="${RESTARTED}"
                  fi
               fi
            fi

            if [ "${info[${PID_INDEX}]}" != "${RESTARTED}" ] ; then
               sm_not_done=true
            fi

            # Add the PID to the process line
            sm_managed_processes[${index}]="${info[${PROCESS_INDEX}]}:${info[${PID_INDEX}]}:${info[${ALIAS_INDEX}]}"

            ((index++))
         done
      fi

      # log when SM restarts are done print a summary only once
      if [ "${sm_not_done}" = false -a "${sm_done}" = false ] ; then
         sm_done=true
         logged=false
         for DAEMON in "${sm_managed_processes[@]}"
         do
            info=(${DAEMON//:/ })
            if [ "${info[${PID_INDEX}]}" == "${RESTARTED}" ] ; then
               if [ "${logged}" = false ] ; then
                   loginfo "The following 'sm managed' processes have been 'restarted'"
                   logged=true
               fi
               loginfo "... process: ${info[${PROCESS_INDEX}]}"
            fi
         done
         logged=false
         for DAEMON in "${sm_managed_processes[@]}"
         do
            info=(${DAEMON//:/ })
            if [ "${info[${PID_INDEX}]}" == "${SKIPPED}" ] ; then
               if [ "${logged}" = false ] ; then
                   loginfo "The following 'sm managed' processes have been 'skipped' ; due to previous restart"
                   logged=true
               fi
               loginfo "... process: ${info[${PROCESS_INDEX}]}"
            fi
         done
      fi
   fi

   #########################################################################
   # For all nodes ....
   #########################################################################

   # Loop over all PMON proceses looking for complete restarts.
   # Update process struct PID field as status is learned.

   if [ "$pmon_done" = false ] ; then
      # Start assuming we are not done
      pmon_not_done=false
      index=0
      for DAEMON in "${pmon_managed_processes[@]}"
      do
         info=(${DAEMON//:/ })

         # Don't wast time on processes that are being skipped due to past restart
         if [ "${info[${PID_INDEX}]}" == "${SKIPPED}" ] ; then
            ((index++))
            continue

         # Don't wast time on processes that have already restarted
         elif [ "${info[${PID_INDEX}]}" == "${RESTARTED}" ] ; then
             ((index++))
             continue

         # Don't look for disabled processes
         elif [ "${info[${PID_INDEX}]}" == "${DISABLED}" ] ; then
             ((index++))
             continue

         # Don't look at not running processes
         elif [ "${info[${PID_INDEX}]}" == "${NOPID}" ] ; then
             ((index++))
             continue
         fi

         # Save the original PID
         new_pid=`pidof ${info[${PROCESS_INDEX}]}`
         if [ $? -eq 0 -a "${new_pid}" != "" ] ; then
            # set the process as restarted as soon as we have a new pid
            if [ "${info[${PID_INDEX}]}" != "${RESTARTED}" ] ; then
               loginfo "${info[${PROCESS_INDEX}]} ${RESTARTED} ok [PID: ${info[${PID_INDEX}]} -> ${new_pid}]"
               info[${PID_INDEX}]=${RESTARTED}
            fi
         fi

         # Set not done as long as there is one process not restarted
         if [ "${info[${PID_INDEX}]}" != "${RESTARTED}" ] ; then
            pmon_not_done=true
         fi

         # Add the PID to the process line
         pmon_managed_processes[${index}]="${info[${PROCESS_INDEX}]}:${info[${PID_INDEX}]}"

         ((index++))
      done
   fi

   # log when all pmond restarts are done
   if [ "${pmon_not_done}" = false -a "${pmon_done}" = false ] ; then
      pmon_done=true
      logged=false
      for DAEMON in "${pmon_managed_processes[@]}"
      do
         info=(${DAEMON//:/ })

         if [ "${info[${PID_INDEX}]}" == "${RESTARTED}" ] ; then
             if [ "${logged}" = false ] ; then
                loginfo "The following 'pmon managed' processes have been 'restarted'"
                logged=true
            fi
            loginfo "... process: ${info[${PROCESS_INDEX}]}"
         fi
      done

      logged=false
      for DAEMON in "${pmon_managed_processes[@]}"
      do
         info=(${DAEMON//:/ })

         if [ "${info[${PID_INDEX}]}" == "${SKIPPED}" ] ; then
             if [ "${logged}" = false ] ; then
                loginfo "The following 'pmon managed' processes have been 'skipped' ; due to previous restart"
                logged=true
            fi
            loginfo "... process: ${info[${PROCESS_INDEX}]}"
         fi
      done
   fi

   # check for done. If this is not met in timeout then fail is returned
   if [ "$sm_done" = true -a "$pmon_done" = true ] ; then
      GLOBAL_RC=$PATCH_STATUS_OK
      break
   fi

   sleep 1
done

loginfo "Maintenance No-Reboot Patching Status: ${GLOBAL_RC}"

exit ${GLOBAL_RC}