#!/bin/bash
#######################################################################
#
# Copyright (c) 2017 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
########################################################################
#
# This file is a new member of the Titanium Cloud "Log Collect Utility".
# This file enhances date restricted collect in response.
#
# This file is invoked by collect_host when a date restricted
# collect using --start-date and/or --end-date options are used.
#
# This new data restricted collect service applies to /var/log and its
# subdirectories only. This service determines if a log file is to be
# included in dated collect by looking at the logs at the head and tail
# of the files and subdirectories in /var/log. Those dates are then
# compared to the user specified date range. if a file is determined to
# contain logs within that date range then that file is included in the
# collect log. A valid log date prefix is "YYYY-MM-DD".
#
# Unfortunately, not all log files contain the correct date placement and
# format. This feature has implemented special case handling for many but not
# not all of such cases. To avoid accidental exclusion of a key file, this
# feature will by default include log files whose log date content could
# not be determined if its file date is after the specified start date.
#
# Note: local convension , example ${head_date} vs ${HEAD_DATE}
#
#       Lower case date variables contain integer values while
#       Upper case date variables contain formatted string values of same.
#
# Calling sequence:
#
# /usr/local/sbin/collect_date <start_date> <end_date> <include_list> <debug>
# /usr/local/sbin/collect_date 20170701 20170901 /tmp/file.list true
#
########################################################################

#
# Import commands, variables and convenience functions available to
# all collectors ; common and user defined.
#
source /usr/local/sbin/collect_utils

# where to find the logs
declare -r baselogdir="/var/log"

# include / exclude labels
declare -r INCLUDE_FILE="inc"
declare -r EXCLUDE_FILE="exc"

# a global reason string that is only valid
# in the context of the file beeing looked at.
declare __this_reason=""

# setup defaults
INC_FILE_LIST="/var/run/collect_include.list"
EXC_FILE_LIST="/var/run/collect_exclude.list"
NOD_FILE_LIST="/var/run/collect_nodate.list"

BOT_DATE="2000-01-01" # beginning of time date
bot_date=730013       # beginning of time date as integer

EOT_DATE="9999-12-31" # end of time date
eot_date=3649810      # end of time date as integer

# manage debug mode
DEBUG="${4}"
set_debug_mode "${DEBUG}"
echo "Debug Mode: ${DEBUG}"

dlog "collect_date args: ${1} ${2} ${3} ${4} ${5}"

#############################################################################
#
# 'track' is the main accounting procedure that manages file inclusions and
#         exclusions as well as the metrics around all the parsed files.
#
#         It also reports accounting mismatch logs, if they occur (should not)
#         and the file that started the mismatch (to assist in debug).
#
# $1 - filename
# $2 - label
#
#############################################################################

# accounting defaults
declare -i file_count=0
declare -i inc_file_count=0
declare -i exc_file_count=0
declare -i empty_file_count=0

function track()
{
    local fn="${1}"
    local label="${2}"

    if [ -z "${fn}" ] ; then
        elog "Ignoring call with empty filename"
        return

    elif [ "${label}" == "totals" ] ; then
        ((file_count++))
        return

    elif [ "${label}" == "empty" ] ; then
        ((empty_file_count++))
        return

    elif [ "${label}" == "${INCLUDE_FILE}" ] ; then
        manage_file "${fn}" "${label}" "${__this_reason}"
        ((inc_file_count++))

    elif [ "${label}" == "${EXCLUDE_FILE}" ] ; then
        manage_file "${fn}" "${label}" "${__this_reason}"
        ((exc_file_count++))

    else
        elog "Unknown label '${label}'"

    fi

    sum=$((inc_file_count + exc_file_count))
    if [ ${file_count} -ne ${sum} ] ; then
        wlog "MISMATCH: ${file_count} != ${inc_file_count} + ${exc_file_count} - ${fn}"
    fi
}

############################################################################
#
# 'summary' is an accounting display procedure used to show the
#           accounting results of the total number of files processed,
#           number of empty files and most importanly the number if
#           included or excluded files.
#
############################################################################

function summary()
{
    dlog "Summary:"
    dlog "Total Files: ${file_count}"
    dlog "Empty Files: ${empty_file_count}"
    dlog "Added Files: ${inc_file_count}"
    dlog "Omitd Files: ${exc_file_count}"
}

#############################################################################
#
# 'date_to_int' converts a standard formatted YYYY-MM-DD string date
#               to an integer and stores it in __this_integer_date variable
#               to be used in context on demand.
#
#############################################################################

# short lived global integer date value updated by date_to_int utility
declare -i __this_integer_date=""

function date_to_int()
{
   local yy="${1:0:4}"
   local mm="${1:5:2}"
   local dd="${1:8:2}"

   # handle leading zeros in month and day
   if [ "${mm:0:1}" == "0" ] ; then
       mm=${mm:1:1}
   fi
   if [ "${dd:0:1}" == "0" ] ; then
       dd=${dd:1:1}
   fi

   # 365 days in a year, 31 days in a month, 1 day in a day
   __this_integer_date=$((yy*365 + mm*31 + dd))
}

############################################################################
#
# 'create_list_file' removes old/stale list file and creates a new empty
#                    one with correct permissions.
#
############################################################################

function create_list_file()
{
    local fn="${1}"
    if [ -e "${fn}" ] ; then
        rm -f "${fn}"
    fi
    touch "${fn}"
    chmod 644 "${fn}"
}

########################################################################
#
# Handle the incoming 'start' and 'end' date format defensively.
#
# If the date is with no dashes as it would come in from the user's
# date specification then set it up like the standard date delimited with '-'
# i.e. 20171002 is updated to 2017-10-02.
#
# If verified to be in the standard format just copy in.
#
# Otherwise assume the start date is from the beginning of time or
# end date is the end of time.

# load up the start date string and integer representation
if [ -z "${1}" ] ; then
    START_DATE="${BOT_DATE}"
elif [[ "${1}" =~ [0-9]{4}[0-9]{2}[0-9]{2} ]] ; then
    START_DATE="${1:0:4}-${1:4:2}-${1:6:2}"
elif [[ "${1}" =~ [0-9]{4}-[0-9]{2}-[0-9]{2} ]] ; then
    START_DATE="${1}"
else
    START_DATE="${BOT_DATE}"
fi

# Convert the correct or corrected 'start' date to an integer value
date_to_int "${START_DATE}"
start_date=${__this_integer_date}


# load up the end date string and integer representation
if [ -z "${2}" ] ; then
    END_DATE="${EOT_DATE}"
elif [[ "${2}" =~ [0-9]{4}[0-9]{2}[0-9]{2} ]] ; then
    END_DATE="${2:0:4}-${2:4:2}-${2:6:2}"
elif [[ "${2}" =~ [0-9]{4}-[0-9]{2}-[0-9]{2} ]] ; then
    END_DATE="${2}"
else
    END_DATE="${EOT_DATE}"
fi

# Convert the correct or corrected 'end' date to an integer value
date_to_int "${END_DATE}"
end_date=${__this_integer_date}

# Handle user error of specifying an end date that is before the start date
if [ ${start_date} -gt ${end_date} ] ; then
    wlog "invalid date range ; end date (${END_DATE}:${end_date}) is before start (${START_DATE}:${start_date})"
    wlog "correcting to defaults: from ${START_DATE} to ${END_DATE}"
    START_DATE="${BOT_DATE}"
    END_DATE="${EOT_DATE}"
    start_date=${bot_date}
    end_date="${eot_date}"
fi

ilog "collecting log files containing logs dated ${START_DATE} to ${END_DATE} (inclusive)"


if [ "${3}" == "" ] ; then
    elog "dated collect include file list name not specified ... exiting"
    exit 1
else
    VAR_LOG_INCLUDE_LIST=${3}
fi

create_list_file "${VAR_LOG_INCLUDE_LIST}"
create_list_file "${INC_FILE_LIST}"
create_list_file "${EXC_FILE_LIST}"
create_list_file "${NOD_FILE_LIST}"

# Declare and init the include and exclude debug lists.
inclist=("")
exclist=("")

#############################################################################
#
# 'filedatelist' is a list of files that are known to not contain dated logs.
#                Instead these files are included unless its file date is
#                older that the specified start date.
#
#############################################################################

filedatelist=("")
filedatelist+=("/var/log/wtmp")
filedatelist+=("/var/log/dmesg")
filedatelist+=("/var/log/dmesg.old")
filedatelist+=("/var/log/sm-trap.log")
filedatelist+=("/var/log/sm-customer.log")
filedatelist+=("/var/log/sm-customer.alarm")
filedatelist+=("/var/log/sm-shutdown.log")
filedatelist+=("/var/log/nfv-vim-events.log")
filedatelist+=("/var/log/fm-customer.log")
filedatelist+=("/var/log/fm-alarm.log")
filedatelist+=("/var/log/lighttpd-access.log")
filedatelist+=("/var/log/audit/audit.log")
filedatelist+=("/var/log/rabbitmq/shutdown_log")
filedatelist+=("/var/log/rabbitmq/startup_log")
filedatelist+=("/var/log/rabbitmq/wait_log")
filedatelist+=("/var/log/rabbitmq/rabbit@localhost.log")
filedatelist+=("/var/log/nfv-vim-alarms.log")
filedatelist+=("/var/log/vswitch.cmds.log")

# This is a list of files to always include
autoaddlist=("")
autoaddlist+=("/var/log/collect.log")

#########################################################################
#
# 'is_in_range' returns true if the specified log file data range
#               is within the bounded date range specified by the caller.
#               Otherwise a false is returned.
#
# ${1} is HEAD_DATE and is the date of the first log of the file in contect
# ${2} is TAIL_DATE and is the date of the last log in the file in context
#
# expected date format is ... YYYY-MM-DD
#
# Calling Sequence is ... is_in_range HEAD_DATE TAIL_DATE
#
# There are several cases that aer handled ;
# see case comment inline below.
#
#########################################################################

function is_in_range()
{
    local HEAD_DATE="${1}"
    local TAIL_DATE="${2}"
    if [[ ${HEAD_DATE} =~ [0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then

        # Convert the date to an integer value
        # to make the compare easier and faster
        date_to_int "${HEAD_DATE}"
        head_date=${__this_integer_date}

        if [[ ${TAIL_DATE} =~ [0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then

            # Convert the date to an integer value
            # to make the compare easier and faster
            date_to_int "${TAIL_DATE}"
            tail_date=${__this_integer_date}

            in_range=false

            # The last log is before the start date or the first log is after the end date
            # if [[ "${TAIL_DATE}" < "${START_DATE}" || "${HEAD_DATE}" > "${END_DATE}" ]] ; then
            if [ ${tail_date} -lt ${start_date} -o ${head_date} -gt ${end_date} ] ; then
                __this_reason+=":case 0"
                in_range=false

            # Case 1: the head after the start but before the end date
            #         .... S ... head ... E ....
            elif [ ${head_date} -ge ${start_date} -a ${head_date} -le ${end_date} ] ; then
                __this_reason+=":case 1"
                in_range=true

            # Case 2: the tail after the start but before the end date
            #         .... S ... tail ... E ....
            elif [ ${tail_date} -ge ${start_date} -a ${tail_date} -le ${end_date} ] ; then
                __this_reason+=":case 2"
                in_range=true

            # Case 3: log file date range spans the start and end dates
            #         head S ...       ... E  tail
            elif [ ${head_date} -le ${start_date} -a ${tail_date} -ge ${end_date} ] ; then
                __this_reason+=":case 3"
                in_range=true

            else
                __this_reason+=":default"
            fi
        else
            __this_reason+=":invalid-tail-date"
            # so the tail date is unknown.
            # include this file as long as the head date is before end date
            if [ ${head_date} -lt ${end_date} ] ; then
                in_range=true
            else
                in_range=false
            fi
        fi

        if [ "${in_range}" = true ] ; then
            __this_reason+=":in-range ${HEAD_DATE} to ${TAIL_DATE}"
            true
        else
            __this_reason+=":out-of-range ${HEAD_DATE} to ${TAIL_DATE}"
            false
        fi
        return
    fi

    __this_reason+=":date-format-error ${HEAD_DATE} to ${TAIL_DATE}"
    true
    return
}

###########################################################################
#
# Name       : want_this_file
#
# Description: This utility first compares the filename to known exception
#              cases and handles them accordingly. Exception cases do look
#              for the date but with different methods. Once the date info
#              is or is not found then the choice to or not to include it
#              follows same general logic as others below.
#
#              If not an exception case then it determines the file type
#              and performs any preprocessing required. i.e. uncompressing
#              the file and switching the filename to the uncompressed name.
#              Data files or other unknown file types are automatically
#              included without further data query by immediately returning
#              true.
#
#              With an expected supported filename in hand this utility will
#              extract the date-only (time not included) portion, the first
#              10 characters of the first and last logs and determin if this
#              logfile has logs that fall withing the specified date range.
#
# Returns    : If there is no valid date found then true is returned.
#              If file contains in range logs then true is returned.
#              if file does not contain in range logs then false is returned.
#
# Parameters : $1 is the full pathed log file name.
#
# $1 - the filename of the file to check the date for
#
###########################################################################

function want_this_file()
{
    local inc=true
    local LOGFILE="${1}"
    local filetype=$(file "${LOGFILE}")
    local HEAD_DATE=""
    local TAIL_DATE=""

    for add in "${autoaddlist[@]}"
    do
        if [ "${add}" == "${LOGFILE}" ] ; then
            __this_reason+="autoadd"
            true
            return
        fi
    done

    ##########################################################################
    # Exception Case: known free formatted log files.
    ##########################################################################
    #
    # Some log files are known to not contain properly dated logs.
    # Such files may just contian free format strings of information.
    #
    # A list of such files is in hard coded in filedatelist.
    # TODO: consider making this a file that is loaded.
    #
    # Check to see if this is an auto add file
    # Only exlude such files if its last modified date is before start date.
    #
    ##########################################################################
    for add in "${filedatelist[@]}"
    do
        if [ "${add}" == "${LOGFILE}" ] ; then
            __this_reason+="filedate"

            # Don't include empty files that are in the hard coded filedatelist
            filetype=$(file "${LOGFILE}")
            if [ ! -z "${filetype}" ] ; then
               case ${filetype} in
                   *empty*)
                      __this_reason="empty"
                      track "${LOGFILE}" "empty"
                      false
                      return
                      ;;
                  *)
                      ;;
               esac
            fi

            # get last modified date
            FILE_DATE=$(stat -c %y "${LOGFILE}" | cut -b 1-10)
            date_to_int "${FILE_DATE}"
            if [ ${__this_integer_date} -ge ${start_date} ] ; then
                __this_reason+=":in-range ${FILE_DATE}"
                true
            else
                __this_reason+=":out-of-range ${FILE_DATE}"
                false
            fi
            return
        fi
    done

    # O.K. if we get here then this filename is not in the static list
    if [ ! -z "${filetype}" ] ; then

        case ${filetype} in

            *directory*)
                # Skip over a directory only path.
                # No worries, the files in that directory will be handled.
                __this_reason+="directory"
                false
                return
                ;;

            *ASCII*|*text*|*compressed*)

                if [[ ${filetype} == *"compressed"* ]] ; then
                    fileext=${LOGFILE##*.}
                    case "${fileext}" in
                        gz)
                            tmpfile=$(mktemp)
                            #__this_reason+="gzipped"
                            zcat "${LOGFILE}" | head -5 > "$tmpfile"
                            zcat "${LOGFILE}" | tail -5 >> "$tmpfile"

                            # save the current compressed log filename
                            # so that it can be restored after the
                            # recursion call below
                            LOGFILE_save="${LOGFILE}"
                            want_this_file "$tmpfile"
                            rc=${?}
                            LOGFILE="${LOGFILE_save}"

                            # cleanup ; get rid of the temp file
                            rm -f "$tmpfile" 2>/dev/null
                            if [ ${rc} -eq 0 ] ; then
                               true
                            else
                               false
                            fi
                            return
                        ;;
                        tgz)
                            __this_reason+="tarball"
                            true
                            return
                            ;;
                        *)
                            __this_reason+="compress:[${fileext}]"
                            true
                            return
                            ;;
                    esac
                fi

                # Read the first log in the file
                HEAD_DATE=$(head -1 "${LOGFILE}")

                ##############################################################
                # Minor Exception Case: empty/short first log
                ##############################################################
                #
                # handle one empty or short first line by fetching second log
                #
                ##############################################################

                if [ ${#HEAD_DATE} -lt 10 ] ; then
                    HEAD_DATE=$(head -2 "${LOGFILE}" | sed -n '2p' | cut -b 1-11)
                fi


                ##############################################################
                # Typical Case: YYYY-MM-DD
                ##############################################################
                #
                # check for most typical date format.
                #
                ##############################################################

                if [[ ${HEAD_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then
                    __this_reason+="typical"
                    TAIL_DATE=$(tail -1 "${LOGFILE}" | cut -b 1-11)
                    if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then

                       # a call to 'is_in_range' returns false (1) if this
                       # file's logs are all out of range date
                       is_in_range "${HEAD_DATE:0:10}" "${TAIL_DATE:0:10}"
                       if [ $? -eq 0 ] ; then
                           true
                       else
                           false
                       fi
                       return

                   else

                       #######################################################
                       # Exception Case: Unrecognized date format in last log
                       #######################################################
                       #
                       # try the second last line. This case is typical in
                       # cron.log in 15.12 MAIL logs which send a purious ')'
                       # as a second log. Also if the log file has auto blank
                       # lines between logs leaving a blank line as the last
                       # log.
                       #
                       # this exception ties the second last log instead.
                       #
                       #######################################################
                       TAIL_DATE=$(tail -2 "${LOGFILE}" | sed -n '1p' | cut -b 1-11)
                       if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then

                           is_in_range "${HEAD_DATE:0:10}" "${TAIL_DATE:0:10}"
                           if [ $? -eq 0 ] ; then
                               true
                           else
                               false
                           fi
                           return

                       else
                           # default to true if the dates could not be parsed
                           __this_reason+=":invalid-tail-date"

                           date_to_int "${HEAD_DATE}"
                           head_date=${__this_integer_date}

                           # so the tail date is unknown.
                           # include this file as long as the head date is before end date
                           if [ ${head_date} -lt ${end_date} ] ; then
                              true
                           else
                              false
                           fi
                           return
                       fi
                   fi

               else

                   ###########################################################
                   # Exception Case 1: logs date prefix starts with '['
                   ###########################################################
                   #
                   # logdate starts with a '[' ... [2017-10-02
                   #
                   # In this case we just recognize it and increment past it
                   # and then assume the last log will have the same format
                   #
                   ###########################################################

                   if [ "${HEAD_DATE:0:1}" == "[" ] ; then
                       __this_reason+="exception1"
                       HEAD_DATE=${HEAD_DATE:1:11}
                       if [[ ${HEAD_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then

                           TAIL_DATE=$(tail -1 "${LOGFILE}" | cut -b 2-11)
                           if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then
                               __this_reason+=".1"
                               is_in_range "${HEAD_DATE:0:10}" "${TAIL_DATE:0:10}"
                               if [ $? -eq 0 ] ; then
                                   true
                               else
                                   false
                               fi
                               return
                           else
                               TAIL_DATE=$(tail -1 "${LOGFILE}" | cut -b 1-10)
                               if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then
                                   __this_reason+=".2"
                                   is_in_range "${HEAD_DATE:0:10}" "${TAIL_DATE:0:10}"
                                   if [ $? -eq 0 ] ; then
                                       true
                                   else
                                       false
                                   fi
                                   return

                               else

                                   if [ "${TAIL_DATE:0:1}" == "[" ] ; then
                                       __this_reason+=".3"
                                       TAIL_DATE=${TAIL_DATE:1:11}
                                       if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then
                                           is_in_range "${HEAD_DATE}" "${TAIL_DATE}"
                                           if [ $? -eq 0 ] ; then
                                               true
                                           else
                                               false
                                           fi
                                           return
                                       else
                                           __this_reason+=":invalid-tail-date"
                                           true
                                           return
                                       fi
                                   else
                                       __this_reason+=":tail-date-not-found"
                                       is_in_range "${HEAD_DATE}" "${EOT_DATE}"
                                       if [ $? -eq 0 ] ; then
                                           true
                                       else
                                           false
                                       fi
                                       return
                                   fi
                               fi
                           fi
                       else
                           # /var/log/dmesg is typical of this case
                           # no log date and many logs start with [   uptime]
                           __this_reason+=":invalid-head-date"
                           true
                           return
                       fi

                   ###########################################################
                   # Exception Case 2: journel.log handling
                   ###########################################################
                   #
                   # first log in file contains start and stop date
                   #
                   # "-- Logs begin at Thu 2017-07-06 12:28:35 UTC, end at Thu 2017-07-06 12:33:31 UTC. --"
                   #                       ^^^^^^^^^^                          ^^^^^^^^^^
                   #
                   # This exception case gets the head and tail log date from
                   # this first log.
                   ###########################################################

                   elif [ "${HEAD_DATE:0:13}" == "-- Logs begin" ] ; then
                       __this_reason+="exception2"

                       # need to get more of the line
                       HEAD_DATE=$(head -1 "${LOGFILE}")

                       is_in_range "${HEAD_DATE:21:10}" "${HEAD_DATE:57:10}"
                       if [ $? -eq 0 ] ; then
                           true
                       else
                           false
                       fi
                       return

                   ###########################################################
                   # Exception Case 3: journel.log handling
                   ###########################################################
                   #
                   # some logs like openstack.log have some logs that are
                   # prefixed by keystone:log. This case handles that
                   #
                   ###########################################################
                   elif [ "${HEAD_DATE:0:13}" == "keystone:log " ] ; then
                       __this_reason+="exception3"

                       # need to get more of the line
                       HEAD_DATE="${HEAD_DATE:13:10}"
                       TAIL_DATE=$(tail -1 "${LOGFILE}")

                       if [ "${TAIL_DATE:0:13}" == "keystone:log " ] ; then
                           TAIL_DATE="${TAIL_DATE:13:10}"
                       else
                           TAIL_DATE="${TAIL_DATE:0:10}"
                       fi

                       is_in_range "${HEAD_DATE}" "${TAIL_DATE}"
                       if [ $? -eq 0 ] ; then
                           true
                       else
                           false
                       fi
                       return

                   else

                       #######################################################
                       # Exception Case 4: horizon.log
                       #######################################################
                       #
                       # Search the first and last 30 logs for a valid date.
                       # This should handle seeing a traceback at the head or
                       # tail of the log file.
                       #
                       #######################################################
                       __this_reason+="exception4"
                       temp_head=$(head -30 "${LOGFILE}")
                       for ((loop_head=1;loop_head<31;loop_head++))
                       do
                           HEAD_DATE=$(echo "${temp_head}" | sed -n "${loop_head}"p | cut -b 1-10)
                           if [[ ${HEAD_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then
                               temp_tail=$(tail -30 "${LOGFILE}")
                               for ((loop_tail=1;loop_tail<31;loop_tail++))
                               do
                                   TAIL_DATE=$(echo "${temp_tail}" | sed -n ${loop_tail}p | cut -b 1-10)
                                   if [[ ${TAIL_DATE} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then

                                       is_in_range "${HEAD_DATE}" "${TAIL_DATE}"
                                       if [ $? -eq 0 ] ; then
                                           true
                                       else
                                           false
                                       fi
                                       return

                                   fi
                               done

                               # default to including it if no date at
                               # the end of the file is found
                               true
                               return
                           fi
                       done

                       ######################################################
                       # Exception Case 5:
                       ######################################################
                       #
                       # Otherwise the file has no date or the date
                       # format is unrecognized so just include the file
                       # regardless of its date.
                       #
                       ######################################################
                       __this_reason="nodate"
                       true
                       return
                   fi
               fi
               ;;

           *archive*)

               # Archive files like .tar are not extracted.
               # Instead it is only collected if its last modified date is
               # after the start date

               __this_reason+="archive"
               FILE_DATE=$(stat -c %y "${LOGFILE}" | cut -b 1-10)
               date_to_int "${FILE_DATE}"
               if [ ${__this_integer_date} -ge ${start_date} ] ; then
                   __this_reason+=":in-range ${FILE_DATE}"
                   true
               else
                   __this_reason+=":out-of-range ${FILE_DATE}"
                   false
               fi
               return
               ;;

           *empty*)
               __this_reason="empty"
               track "${LOGFILE}" "empty"
               false
               return
               ;;

           *data*)
               __this_reason="data"
               true
               return
               ;;

           *executable*)
               __this_reason="executable"
               true
               return
               ;;

           # very short file (no magic)
           *"very short file"*)
               __this_reason="small"
               true
               return
               ;;

           *link*)
               __this_reason="link"
               false
               return
               ;;

           *swap*)

               __this_reason="swap"
               false
               return
               ;;

           *fifo*)

               __this_reason="fifo"
               false
               return
               ;;

           *socket*)

               __this_reason="socket"
               false
               return
               ;;

           *)
               __this_reason="other"
               true
               return
               ;;
        esac
    else
        __this_reason="unknown"
        wlog "Adding ${logfile} ; unknown filetype"
        true
        return
    fi

    # catch all default
    true
    return
}

#############################################################################
#
# 'manage_file' adds the specified file to either the 'include' or exclude'
#               reason lists. In the include case the most important part of
#               this function appends the filename to the file specified by
#               "VAR_LOG_INCLUDE_LIST" which is the file that collect_host
#               uses to know what files in /var/log need to be included in
#               the collect tarball.
#
#############################################################################

function manage_file()
{
    local filename="${1}"
    local action="${2}"
    local reason="${3}"

    if [ "${action}" == "${EXCLUDE_FILE}" ] ; then
        echo "${filename} excluded (${reason})" >> "${EXC_FILE_LIST}"
    else
        echo "${filename} included (${reason})" >> "${INC_FILE_LIST}"

        # add the file to the list of files to be collected
        echo "${filename}" >> ${VAR_LOG_INCLUDE_LIST}
    fi

    dlog "${action}: ${filename} (${reason})"
}

#############################################################################
#
# 'handle_response' adds or excludes the specified file based on
#                   arguement $2 being  0 - true  - include or
#                                      !0 - false - exclude
#
# $1 - file
# $2 - include control ( true or false )
#
#############################################################################

function handle_response()
{
    local logfile="${1}"
    local include="${2}"

    if [ "${include}" -eq 0 ] ; then
        inclist=("${inclist[@]}" ${logfile})
        track "${logfile}" "${INCLUDE_FILE}"

    else
        exclist=("${exclist[@]}" ${logfile})
        track "${logfile}" "${EXCLUDE_FILE}"
    fi

    # record any that have been tagged as 'nodate' as
    # candidate for special handling.
    if [[ "${__this_reason}" == *"nodate"* ]] ; then
        echo "${logfile}" >> "${NOD_FILE_LIST}"
    fi
}

###########################################################################
###########################################################################
#
#             Lets start looking at the files now ...
#
# Get all the files in /var/log base dir (not the subdirectories)
#
###########################################################################
###########################################################################

# get a list of the files in "baselogdir" ; aka /var/log
# will look at the sub directories later.
dirlist+=$(find ${baselogdir} -mindepth 1 -maxdepth 1 -type f)

#
# Debug:
#
# To debug handling a specific file as a filelist override.
# This clears the list in favor of the specific file specified as
# argument 8 on the command line.
#
if [ "${5}" != "" ] ; then
    dlog "Overriding dirlist with specified file:${5}"
    dirlist=("${5}")
fi

# echo "${baselogdir} filelist: ... ${dirlist}..."
for logfile in ${dirlist}
do
    # echo "File: ${logfile}"
    __this_reason=""
    track "${logfile}" "totals"
    want_this_file "${logfile}"
    handle_response "${logfile}" "${?}"
done

###########################################################################
#            Get all the files in baselogdir subdirectories               #
###########################################################################

subdirlist=$(find ${baselogdir} -mindepth 1 -maxdepth 20 -type d)

#
# Debug:
#
# To debug handling a specific file that is in a /var/log subdirectory as a
# filelist override.
#
if [ "${5}" != "" ] ; then
    dlog "Overriding subdirlist with specified file:${5}"
    subdirlist=("")
fi

# echo "${baselogdir} subdirlist ${subdirlist}..."
for logdir in ${subdirlist}
do
    __this_reason=""

    # this find must find more than just its own dir
    # so we compare to greater than one
    if [ $(find "${logdir}" | wc -l) -gt 1 ]; then
        for logfile in ${logdir}/*
        do
            __this_reason=""
            track "$logfile" "totals"
            want_this_file "$logfile"
            handle_response "$logfile" "$?"
        done
    else
        __this_reason="empty"
        manage_file "${logdir}" "${EXCLUDE_FILE}" "empty directory"
    fi
done


dlog "Include List: ${INC_FILE_LIST}"
for inc in "${inclist[@]}"
do
    if [ ${#inc} -gt 2 ] ; then
        dlog "including ${inc}"
        # echo "${inc}" >> "${INC_FILE_LIST}.summary"
    fi
done


dlog "Exclude List: ${EXC_FILE_LIST}"
for exc in "${exclist[@]}"
do
    if [ ${#exc} -gt 2 ] ; then
        dlog "excluding ${exc}"
        # echo "${exc}" >> "${EXC_FILE_LIST}.summary"
    fi
done

summary

exit 0