nfv/nfv/nfv-vim/nfv_vim/dor/_dor_module.py

#
# Copyright (c) 2015-2016 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import os.path

from nfv_common import config
from nfv_common import debug
from nfv_common import timers
from nfv_common.helpers import coroutine
from nfv_common.helpers import local_uptime_in_secs
from nfv_common.helpers import process_uptime_in_secs

from nfv_vim import alarm
from nfv_vim import event_log
from nfv_vim import nfvi
from nfv_vim import objects
from nfv_vim import tables

DLOG = debug.debug_get_logger('nfv_vim.dor')

_alarm_data = None
_minimum_hosts = 0
_dor_stabilized = False
_dor_completed = False
_dor_process_uptime = 0
_dor_stabilize_uptime = 0
_dor_complete_uptime = 0
_dor_complete_percentage = 0
_system_state_get_inprogress = False
_system_state_gathered = False

_process_start_timestamp_ms = timers.get_monotonic_timestamp_in_ms()

NFV_VIM_DOR_COMPLETE_FILE = '/var/run/.nfv-vim.dor_complete'


@coroutine
def _system_state_query_callback():
    """
    System state query callback
    """
    global _alarm_data
    global _minimum_hosts, _dor_stabilized, _dor_completed
    global _dor_complete_percentage
    global _system_state_get_inprogress, _system_state_gathered

    response = (yield)
    DLOG.info("System-State callback, response=%s." % response)

    _system_state_get_inprogress = False

    if response['completed']:
        result_data = response.get('result-data', None)
        if result_data is not None:
            host_table = tables.tables_get_host_table()
            total_hosts = 0
            dor_complete_hosts = 0

            for host_data in result_data:
                host = host_table.get(host_data['hostname'], None)
                if host is not None:
                    if objects.HOST_PERSONALITY.WORKER in host.personality:
                        DLOG.info("Host %s uptime is %s, host_uuid=%s."
                                  % (host_data['hostname'], host_data['uptime'],
                                     host_data['uuid']))

                        total_hosts += 1
                        if host_data['uptime'] >= _dor_complete_uptime:
                            dor_complete_hosts += 1

            if 0 < total_hosts:
                completion_percentage = (dor_complete_hosts * 100 / total_hosts)
            else:
                completion_percentage = 0

            if _dor_complete_percentage <= completion_percentage:
                _dor_completed = True

            DLOG.info("DOR host completion percentage is %s, threshold=%s."
                      % (completion_percentage, _dor_complete_percentage))

            if not _dor_completed and total_hosts < _minimum_hosts:
                DLOG.info("DOR total hosts %s is less than minimum hosts %s."
                          % (total_hosts, _minimum_hosts))

                if process_uptime_in_secs() >= _dor_process_uptime:
                    _dor_completed = True

                elif not _dor_stabilized:
                    _dor_stabilized = True
                    DLOG.info("DOR stabilized.")

        if _dor_completed:
            open(NFV_VIM_DOR_COMPLETE_FILE, 'w').close()
            _dor_stabilized = True
            if _alarm_data is not None:
                alarm.clear_general_alarm(_alarm_data)
                event_log.issue_general_log(
                    event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_EXIT)
                _alarm_data = None
            DLOG.info("DOR completed.")
            _system_state_gathered = True

        elif _alarm_data is None:
            _alarm_data = \
                alarm.raise_general_alarm(alarm.ALARM_TYPE.MULTI_NODE_RECOVERY_MODE)
            event_log.issue_general_log(
                event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_ENTER)


@coroutine
def _dor_timer():
    """
    DOR timer
    """
    global _alarm_data
    global _dor_stabilized, _dor_completed
    global _system_state_get_inprogress

    while not _dor_completed:
        (yield)

        if _dor_completed:
            break

        if os.path.exists(NFV_VIM_DOR_COMPLETE_FILE):
            _dor_stabilized = True
            _dor_completed = True
            if _alarm_data is not None:
                alarm.clear_general_alarm(_alarm_data)
                event_log.issue_general_log(
                    event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_EXIT)
                _alarm_data = None
            DLOG.info("DOR completed.")
            break

        if local_uptime_in_secs() > _dor_complete_uptime:
            open(NFV_VIM_DOR_COMPLETE_FILE, 'w').close()
            _dor_stabilized = True
            _dor_completed = True
            if _alarm_data is not None:
                alarm.clear_general_alarm(_alarm_data)
                event_log.issue_general_log(
                    event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_EXIT)
                _alarm_data = None
            DLOG.info("DOR completed.")
            break

        now_ms = timers.get_monotonic_timestamp_in_ms()
        elapsed_secs = (now_ms - _process_start_timestamp_ms) / 1000

        if not _dor_stabilized and elapsed_secs > _dor_stabilize_uptime:
            _dor_stabilized = True
            DLOG.info("DOR stabilized.")

        if not (_system_state_get_inprogress or _system_state_gathered):
            nfvi.nfvi_get_system_state(_system_state_query_callback())
            _system_state_get_inprogress = True


def system_is_stabilized():
    """
    Returns true if system is stabilized after a DOR
    """
    return _dor_stabilized


def dor_is_complete():
    """
    Returns true if DOR is complete
    """
    return _dor_completed


def dor_initialize():
    """
    Initialize DOR handling
    """
    global _minimum_hosts, _dor_process_uptime, _dor_stabilize_uptime
    global _dor_complete_uptime, _dor_complete_percentage
    global _system_state_get_inprogress, _system_state_gathered

    if config.section_exists('dor-configuration'):
        section = config.CONF['dor-configuration']
        _minimum_hosts = int(section.get('minimum_hosts', 4))
        _dor_process_uptime = int(section.get('dor_process_uptime', 60))
        _dor_stabilize_uptime = int(section.get('dor_stabilize_uptime', 240))
        _dor_complete_uptime = int(section.get('dor_complete_uptime', 1200))
        _dor_complete_percentage = int(section.get('dor_complete_uptime', 50))
    else:
        _minimum_hosts = 4
        _dor_process_uptime = 60
        _dor_stabilize_uptime = 240
        _dor_complete_uptime = 1200
        _dor_complete_percentage = 50

    _system_state_get_inprogress = False
    _system_state_gathered = False

    timers.timers_create_timer('dor', 1, 20, _dor_timer)

    # Uncomment the following for testing without the need to reset the
    # controller.  Also need to remove /var/run/.nfv-vim.dor_complete
    # _minimum_hosts = 0
    # _dor_complete_uptime += local_uptime_in_secs()


def dor_finalize():
    """
    Finalize DOR handling
    """
    pass