metal/mtce/src/scripts/crashDumpMgr

#!/bin/bash
#
# Copyright (c) 2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# Modify it that is to support the debian coredump file.
# coredump files are dmesg.202206101633 and dump.202206101633 in Debian.

CRASHDUMPMGR_TAG=${CRASHDUMPMGR_TAG:-"crashDumpMgr"}

RETVAL=0

max_size=3221225472       # "3GiB"
min_remainder=1073741824  # "1GiB"

# number format to/from human readable commands.
NUMFMT_TO_HR="/usr/bin/numfmt --to=iec"
NUMFMT_FROM_HR="/usr/bin/numfmt --from=auto"

#############################################################################
# Log message to syslog
#############################################################################

function log()
{
    logger -t "${CRASHDUMPMGR_TAG}" "$@"
}

#############################################################################
#
# Name   : manage_crash_dumps
#
# Purpose: Prevent crash dumps from filling up the root fs
#
# The kernel directs new crash dump bundles to
# /var/crash/<dated vmcore bundle>. Crash dump
# bundles are quite large and, if too many occur,
# can fill up its target filesystem.
#
# This function nicely tars a crash bundle found in /var/crash
# to /var/log/crash.
#
# The first bundle is tar'ed as vmcore_first.tar and preserved.
# Subsequent crash bundles are nicely tar'ed as vmcore.tar
#
# Save the crash dump vmcore summary for all crash dumps.
#
# Assumptions: logration is used to compress these bundles in the background
#
# Parameters : $1 = max_size ; maximum vmcore size to keep
#
############################################################################

function manage_crash_dumps()
{
    if [ "${1}" != "" ] ; then
        max_size=${1}
        log "max_size=$max_size"
    fi

    CRASH_DIR="/var/crash"
    CRASH_BUNDLE_DIR="/var/log/crash"
    OTHER_BUNDLE="${CRASH_BUNDLE_DIR}/vmcore.tar"
    FIRST_BUNDLE="${CRASH_BUNDLE_DIR}/vmcore_first.tar"
    FIRST_BUNDLE_ROTATED="${CRASH_BUNDLE_DIR}/vmcore_first.tar.1.gz"
    CRASH_BUNDLE_SUMMARY="vmcore-dmesg.txt"
    CRASH_BUNDLE_SUMMARY_DEB="dmesg."

    # tar command and nice levels
    TAR_CMD="tar -cf"
    NICE_CMD="/usr/bin/nice -n19"
    IONICE_CMD="/usr/bin/ionice -c2 -n7"

    log "managing ${CRASH_DIR}"
    cleanup=false

    # create dir if it does not exist
    if [ ! -d ${CRASH_BUNDLE_DIR} ] ; then
        mkdir ${CRASH_BUNDLE_DIR}
    fi

    for entry in ${CRASH_DIR}/*
    do
        remove_entry=false
        if [ -d "${entry}" ] ; then
            time=${entry##*/}
            if [ -e "${entry}/${CRASH_BUNDLE_SUMMARY_DEB}${time}" ] ; then
                log "saving summary: ${CRASH_DIR}/$(basename ${time})_${CRASH_BUNDLE_SUMMARY_DEB}${time}"

                # save the crash dump dmesg.<date> for debian for all crash dumps
                cp -a ${entry}/${CRASH_BUNDLE_SUMMARY_DEB}${time} ${CRASH_DIR}/$(basename ${time})_${CRASH_BUNDLE_SUMMARY_DEB}${time}
            fi
            if [ -e "${entry}/${CRASH_BUNDLE_SUMMARY}" ] ; then
                log "saving summary: ${CRASH_DIR}/$(basename ${entry})_${CRASH_BUNDLE_SUMMARY}"

                # save the crash dump vmcore summary for all crash dumps
                cp -a ${entry}/${CRASH_BUNDLE_SUMMARY} ${CRASH_DIR}/$(basename ${entry})_${CRASH_BUNDLE_SUMMARY}
            fi
            if [ -e "${entry}/dump.${time}" ] || [ -e "${entry}/vmcore" ] ; then

                # get the size of this vmcore file ; raw and human readable
                if [ -e "${entry}/dump.${time}" ] ; then
                    vmcore_size=$(stat --format='%s' ${entry}/dump.${time})
                else
                    vmcore_size=$(stat --format='%s' ${entry}/vmcore)
                fi
                vmcore_size_hr=$(${NUMFMT_TO_HR} ${vmcore_size})

                # get available ${CRASH_BUNDLE_DIR} fs space in 1k blocks and convert that to bytes
                available=$(($(df -k ${CRASH_BUNDLE_DIR}  | grep -v Available | awk '{ print $4 }')*1000))
                available_hr=$(${NUMFMT_TO_HR} ${available})
                log "new vmcore detected (size:${vmcore_size}:${vmcore_size_hr}) ; ${CRASH_BUNDLE_DIR} avail:${available}:${available_hr}"

                # Don't save this crash dump if it would leave the
                # ${CRASH_BUNDLE_DIR} filesystem with less than 1GiB.
                if [ ${available} -gt ${vmcore_size} ]; then
                    remaining=$((available-vmcore_size))
                else
                    remaining=0
                fi

                if [ "${cleanup}" = true ] ; then
                    log "... remove ${entry} ; cleanup"
                    remove_entry=true

                # check for min required 'remaining' ${CRASH_BUNDLE_DIR} filesystem space
                elif [ ${remaining} -lt ${min_remainder} ] ; then
                    log "insufficient space in ${CRASH_BUNDLE_DIR} for ${vmcore_size_hr} ${entry}; would leave only ${remaining} bytes"
                    remove_entry=true

                # create a new crash bundle if the vmcore file isn't oversized
                elif [ ${vmcore_size} -lt ${max_size} ] ; then
                    if [ -e ${FIRST_BUNDLE} -o -e ${FIRST_BUNDLE_ROTATED} ] ; then
                        if [ ! -e ${OTHER_BUNDLE} ] ; then
                            log "creating bundle from ${entry}"
                            ${IONICE_CMD} ${NICE_CMD} ${TAR_CMD} ${OTHER_BUNDLE} -C ${CRASH_DIR} $(basename ${entry})
                            cleanup=true
                        fi
                    else
                        log "creating first bundle from ${entry}"
                        ${IONICE_CMD} ${NICE_CMD} ${TAR_CMD} ${FIRST_BUNDLE} -C ${CRASH_DIR} $(basename ${entry})
                        cleanup=true
                    fi
                    remove_entry=true
                else
                    log "deleting oversize (${vmcore_size_hr}) vmcore file $(basename ${entry})"
                    remove_entry=true
                fi
            elif [[ "$entry" == *"_dmesg."* ]] || [[ "$entry" == *"_vmcore-dmesg.txt"* ]] ; then
                log "saved old $entry summary"
            elif [[ "$entry" != "$CRASH_DIR/*"  ]] ; then
                # removes vmcore files not named properly
                # i.e vmcore.incomplete
                remove_entry=true
            fi
        elif [[ "$entry" != *"_dmesg."* ]] && [[ "$entry" != *"_vmcore-dmesg.txt"* ]] ; then
            # removes files in /var/crash that are not crash dumps related
            remove_entry=true
        fi

        if [ "${remove_entry}" = true ] ; then
            log "removing ${entry}"
            rm -rf "${entry}"
        fi
    done
}


function print_help()
{
    echo "$(basename $0) { --max-size <human-readable-size> }"
}

# Parse the command line
while [[ ${#} -gt 0 ]] ; do

    key="${1}"

    case $key in

        -h|--help)
            print_help
            ;;

        --max-size)
            max_size=$(echo "$2" | ${NUMFMT_FROM_HR})
            log "max crash dump vmcore size is ${2} (${max_size})"
            shift
            ;;

        *)
            print_help
            ;;
    esac
    shift
done

manage_crash_dumps $max_size

exit $RETVAL