216 lines
6.8 KiB
Python
Executable File
216 lines
6.8 KiB
Python
Executable File
#
|
|
# Copyright (c) 2015-2016 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
import os.path
|
|
|
|
from nfv_common import config
|
|
from nfv_common import debug
|
|
from nfv_common import timers
|
|
|
|
from nfv_common.helpers import coroutine
|
|
from nfv_common.helpers import local_uptime_in_secs
|
|
from nfv_common.helpers import process_uptime_in_secs
|
|
|
|
from nfv_vim import alarm
|
|
from nfv_vim import event_log
|
|
from nfv_vim import nfvi
|
|
from nfv_vim import objects
|
|
from nfv_vim import tables
|
|
|
|
DLOG = debug.debug_get_logger('nfv_vim.dor')
|
|
|
|
_alarm_data = None
|
|
_minimum_hosts = 0
|
|
_dor_stabilized = False
|
|
_dor_completed = False
|
|
_dor_process_uptime = 0
|
|
_dor_stabilize_uptime = 0
|
|
_dor_complete_uptime = 0
|
|
_dor_complete_percentage = 0
|
|
_system_state_get_inprogress = False
|
|
_system_state_gathered = False
|
|
|
|
_process_start_timestamp_ms = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
NFV_VIM_DOR_COMPLETE_FILE = '/var/run/.nfv-vim.dor_complete'
|
|
|
|
|
|
@coroutine
|
|
def _system_state_query_callback():
|
|
"""
|
|
System state query callback
|
|
"""
|
|
global _alarm_data
|
|
global _minimum_hosts, _dor_stabilized, _dor_completed
|
|
global _dor_complete_percentage
|
|
global _system_state_get_inprogress, _system_state_gathered
|
|
|
|
response = (yield)
|
|
DLOG.info("System-State callback, response=%s." % response)
|
|
|
|
_system_state_get_inprogress = False
|
|
|
|
if response['completed']:
|
|
result_data = response.get('result-data', None)
|
|
if result_data is not None:
|
|
host_table = tables.tables_get_host_table()
|
|
total_hosts = 0
|
|
dor_complete_hosts = 0
|
|
|
|
for host_data in result_data:
|
|
host = host_table.get(host_data['hostname'], None)
|
|
if host is not None:
|
|
if objects.HOST_PERSONALITY.WORKER in host.personality:
|
|
DLOG.info("Host %s uptime is %s, host_uuid=%s."
|
|
% (host_data['hostname'], host_data['uptime'],
|
|
host_data['uuid']))
|
|
|
|
total_hosts += 1
|
|
if host_data['uptime'] >= _dor_complete_uptime:
|
|
dor_complete_hosts += 1
|
|
|
|
if 0 < total_hosts:
|
|
completion_percentage = (dor_complete_hosts * 100 / total_hosts)
|
|
else:
|
|
completion_percentage = 0
|
|
|
|
if _dor_complete_percentage <= completion_percentage:
|
|
_dor_completed = True
|
|
|
|
DLOG.info("DOR host completion percentage is %s, threshold=%s."
|
|
% (completion_percentage, _dor_complete_percentage))
|
|
|
|
if not _dor_completed and total_hosts < _minimum_hosts:
|
|
DLOG.info("DOR total hosts %s is less than minimum hosts %s."
|
|
% (total_hosts, _minimum_hosts))
|
|
|
|
if process_uptime_in_secs() >= _dor_process_uptime:
|
|
_dor_completed = True
|
|
|
|
elif not _dor_stabilized:
|
|
_dor_stabilized = True
|
|
DLOG.info("DOR stabilized.")
|
|
|
|
if _dor_completed:
|
|
open(NFV_VIM_DOR_COMPLETE_FILE, 'w').close()
|
|
_dor_stabilized = True
|
|
if _alarm_data is not None:
|
|
alarm.clear_general_alarm(_alarm_data)
|
|
event_log.issue_general_log(
|
|
event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_EXIT)
|
|
_alarm_data = None
|
|
DLOG.info("DOR completed.")
|
|
_system_state_gathered = True
|
|
|
|
elif _alarm_data is None:
|
|
_alarm_data = \
|
|
alarm.raise_general_alarm(alarm.ALARM_TYPE.MULTI_NODE_RECOVERY_MODE)
|
|
event_log.issue_general_log(
|
|
event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_ENTER)
|
|
|
|
|
|
@coroutine
|
|
def _dor_timer():
|
|
"""
|
|
DOR timer
|
|
"""
|
|
global _alarm_data
|
|
global _dor_stabilized, _dor_completed
|
|
global _system_state_get_inprogress
|
|
|
|
while not _dor_completed:
|
|
(yield)
|
|
|
|
if _dor_completed:
|
|
break
|
|
|
|
if os.path.exists(NFV_VIM_DOR_COMPLETE_FILE):
|
|
_dor_stabilized = True
|
|
_dor_completed = True
|
|
if _alarm_data is not None:
|
|
alarm.clear_general_alarm(_alarm_data)
|
|
event_log.issue_general_log(
|
|
event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_EXIT)
|
|
_alarm_data = None
|
|
DLOG.info("DOR completed.")
|
|
break
|
|
|
|
if local_uptime_in_secs() > _dor_complete_uptime:
|
|
open(NFV_VIM_DOR_COMPLETE_FILE, 'w').close()
|
|
_dor_stabilized = True
|
|
_dor_completed = True
|
|
if _alarm_data is not None:
|
|
alarm.clear_general_alarm(_alarm_data)
|
|
event_log.issue_general_log(
|
|
event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_EXIT)
|
|
_alarm_data = None
|
|
DLOG.info("DOR completed.")
|
|
break
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
elapsed_secs = (now_ms - _process_start_timestamp_ms) / 1000
|
|
|
|
if not _dor_stabilized and elapsed_secs > _dor_stabilize_uptime:
|
|
_dor_stabilized = True
|
|
DLOG.info("DOR stabilized.")
|
|
|
|
if not (_system_state_get_inprogress or _system_state_gathered):
|
|
nfvi.nfvi_get_system_state(_system_state_query_callback())
|
|
_system_state_get_inprogress = True
|
|
|
|
|
|
def system_is_stabilized():
|
|
"""
|
|
Returns true if system is stabilized after a DOR
|
|
"""
|
|
return _dor_stabilized
|
|
|
|
|
|
def dor_is_complete():
|
|
"""
|
|
Returns true if DOR is complete
|
|
"""
|
|
return _dor_completed
|
|
|
|
|
|
def dor_initialize():
|
|
"""
|
|
Initialize DOR handling
|
|
"""
|
|
global _minimum_hosts, _dor_process_uptime, _dor_stabilize_uptime
|
|
global _dor_complete_uptime, _dor_complete_percentage
|
|
global _system_state_get_inprogress, _system_state_gathered
|
|
|
|
if config.section_exists('dor-configuration'):
|
|
section = config.CONF['dor-configuration']
|
|
_minimum_hosts = int(section.get('minimum_hosts', 4))
|
|
_dor_process_uptime = int(section.get('dor_process_uptime', 60))
|
|
_dor_stabilize_uptime = int(section.get('dor_stabilize_uptime', 240))
|
|
_dor_complete_uptime = int(section.get('dor_complete_uptime', 1200))
|
|
_dor_complete_percentage = int(section.get('dor_complete_uptime', 50))
|
|
else:
|
|
_minimum_hosts = 4
|
|
_dor_process_uptime = 60
|
|
_dor_stabilize_uptime = 240
|
|
_dor_complete_uptime = 1200
|
|
_dor_complete_percentage = 50
|
|
|
|
_system_state_get_inprogress = False
|
|
_system_state_gathered = False
|
|
|
|
timers.timers_create_timer('dor', 1, 20, _dor_timer)
|
|
|
|
# Uncomment the following for testing without the need to reset the
|
|
# controller. Also need to remove /var/run/.nfv-vim.dor_complete
|
|
# _minimum_hosts = 0
|
|
# _dor_complete_uptime += local_uptime_in_secs()
|
|
|
|
|
|
def dor_finalize():
|
|
"""
|
|
Finalize DOR handling
|
|
"""
|
|
pass
|