359 lines
14 KiB
Python
Executable File
359 lines
14 KiB
Python
Executable File
#
|
|
# Copyright (c) 2018 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
import os
|
|
import subprocess
|
|
|
|
from controllerconfig import backup_restore
|
|
|
|
from fm_api import fm_api
|
|
|
|
from sysinv.common import ceph
|
|
from sysinv.common import constants
|
|
from sysinv.common import utils
|
|
from sysinv.common.fm import fmclient
|
|
from sysinv.common.storage_backend_conf import StorageBackendConfig
|
|
from sysinv.api.controllers.v1 import patch_api
|
|
from sysinv.api.controllers.v1 import vim_api
|
|
from sysinv.openstack.common import log
|
|
from sysinv.openstack.common.gettextutils import _
|
|
|
|
import cgcs_patch.constants as patch_constants
|
|
|
|
LOG = log.getLogger(__name__)
|
|
|
|
|
|
class Health(object):
|
|
|
|
SUCCESS_MSG = _('OK')
|
|
FAIL_MSG = _('Fail')
|
|
|
|
def __init__(self, dbapi):
|
|
self._dbapi = dbapi
|
|
self._ceph = ceph.CephApiOperator()
|
|
|
|
def _check_hosts_provisioned(self, hosts):
|
|
"""Checks that each host is provisioned"""
|
|
provisioned_hosts = []
|
|
unprovisioned_hosts = 0
|
|
for host in hosts:
|
|
if host['invprovision'] != constants.PROVISIONED or \
|
|
host['hostname'] is None:
|
|
unprovisioned_hosts = unprovisioned_hosts + 1
|
|
else:
|
|
provisioned_hosts.append(host)
|
|
|
|
return unprovisioned_hosts, provisioned_hosts
|
|
|
|
def _check_hosts_enabled(self, hosts):
|
|
"""Checks that each host is enabled and unlocked"""
|
|
offline_host_list = []
|
|
for host in hosts:
|
|
if host['administrative'] != constants.ADMIN_UNLOCKED or \
|
|
host['operational'] != constants.OPERATIONAL_ENABLED:
|
|
offline_host_list.append(host.hostname)
|
|
|
|
success = not offline_host_list
|
|
return success, offline_host_list
|
|
|
|
def _check_hosts_config(self, hosts):
|
|
"""Checks that the applied and target config match for each host"""
|
|
config_host_list = []
|
|
for host in hosts:
|
|
if (host.config_target and
|
|
host.config_applied != host.config_target):
|
|
config_host_list.append(host.hostname)
|
|
|
|
success = not config_host_list
|
|
return success, config_host_list
|
|
|
|
def _check_patch_current(self, hosts):
|
|
"""Checks that each host is patch current"""
|
|
system = self._dbapi.isystem_get_one()
|
|
response = patch_api.patch_query_hosts(token=None, timeout=60,
|
|
region_name=system.region_name)
|
|
patch_hosts = response['data']
|
|
not_patch_current_hosts = []
|
|
hostnames = []
|
|
for host in hosts:
|
|
hostnames.append(host['hostname'])
|
|
|
|
for host in patch_hosts:
|
|
# There may be instances where the patching db returns
|
|
# hosts that have been recently deleted. We will continue if a host
|
|
# is the patching db but not sysinv
|
|
try:
|
|
hostnames.remove(host['hostname'])
|
|
except ValueError:
|
|
LOG.info('Host %s found in patching but not in sysinv. '
|
|
'Continuing' % host['hostname'])
|
|
else:
|
|
if not host['patch_current']:
|
|
not_patch_current_hosts.append(host['hostname'])
|
|
|
|
success = not not_patch_current_hosts and not hostnames
|
|
return success, not_patch_current_hosts, hostnames
|
|
|
|
def _check_alarms(self, context, force=False):
|
|
"""Checks that no alarms are active"""
|
|
db_alarms = fmclient(context).alarm.list(include_suppress=True)
|
|
|
|
success = True
|
|
allowed = 0
|
|
affecting = 0
|
|
# Only fail if we find alarms past their affecting threshold
|
|
for db_alarm in db_alarms:
|
|
if isinstance(db_alarm, tuple):
|
|
alarm = db_alarm[0]
|
|
mgmt_affecting = db_alarm[constants.DB_MGMT_AFFECTING]
|
|
else:
|
|
alarm = db_alarm
|
|
mgmt_affecting = db_alarm.mgmt_affecting
|
|
if fm_api.FaultAPIs.alarm_allowed(alarm.severity, mgmt_affecting):
|
|
allowed += 1
|
|
if not force:
|
|
success = False
|
|
else:
|
|
affecting += 1
|
|
success = False
|
|
|
|
return success, allowed, affecting
|
|
|
|
def get_alarms_degrade(self, context, alarm_ignore_list=[],
|
|
entity_instance_id_filter=""):
|
|
"""Return all the alarms that cause the degrade"""
|
|
db_alarms = fmclient(context).alarm.list(include_suppress=True)
|
|
degrade_alarms = []
|
|
|
|
for db_alarm in db_alarms:
|
|
if isinstance(db_alarm, tuple):
|
|
alarm = db_alarm[0]
|
|
degrade_affecting = db_alarm[constants.DB_DEGRADE_AFFECTING]
|
|
else:
|
|
alarm = db_alarm
|
|
degrade_affecting = db_alarm.degrade_affecting
|
|
# Ignore alarms that are part of the ignore list sent as parameter
|
|
# and also filter the alarms bases on entity instance id.
|
|
# If multiple alarms with the same ID exist, we only return the ID
|
|
# one time.
|
|
if not fm_api.FaultAPIs.alarm_allowed(alarm.severity, degrade_affecting):
|
|
if (entity_instance_id_filter in alarm.entity_instance_id and
|
|
alarm.alarm_id not in alarm_ignore_list and
|
|
alarm.alarm_id not in degrade_alarms):
|
|
degrade_alarms.append(alarm.alarm_id)
|
|
return degrade_alarms
|
|
|
|
def _check_ceph(self):
|
|
"""Checks the ceph health status"""
|
|
return self._ceph.ceph_status_ok()
|
|
|
|
def _check_license(self, version):
|
|
"""Validates the current license is valid for the specified version"""
|
|
check_binary = "/usr/bin/sm-license-check"
|
|
license_file = '/etc/platform/.license'
|
|
system = self._dbapi.isystem_get_one()
|
|
system_type = system.system_type
|
|
system_mode = system.system_mode
|
|
|
|
with open(os.devnull, "w") as fnull:
|
|
try:
|
|
subprocess.check_call([check_binary, license_file, version,
|
|
system_type, system_mode],
|
|
stdout=fnull, stderr=fnull)
|
|
except subprocess.CalledProcessError:
|
|
return False
|
|
|
|
return True
|
|
|
|
def _check_required_patches(self, patch_list):
|
|
"""Validates that each patch provided is applied on the system"""
|
|
system = self._dbapi.isystem_get_one()
|
|
response = patch_api.patch_query(token=None, timeout=60,
|
|
region_name=system.region_name)
|
|
query_patches = response['pd']
|
|
applied_patches = []
|
|
for patch_key in query_patches:
|
|
patch = query_patches[patch_key]
|
|
patchstate = patch.get('patchstate', None)
|
|
if patchstate == patch_constants.APPLIED or \
|
|
patchstate == patch_constants.COMMITTED:
|
|
applied_patches.append(patch_key)
|
|
|
|
missing_patches = []
|
|
for required_patch in patch_list:
|
|
if required_patch not in applied_patches:
|
|
missing_patches.append(required_patch)
|
|
|
|
success = not missing_patches
|
|
return success, missing_patches
|
|
|
|
def _check_running_instances(self, host):
|
|
"""Checks that no instances are running on the host"""
|
|
|
|
vim_resp = vim_api.vim_host_get_instances(
|
|
None,
|
|
host['uuid'],
|
|
host['hostname'],
|
|
constants.VIM_DEFAULT_TIMEOUT_IN_SECS)
|
|
running_instances = vim_resp['instances']
|
|
|
|
success = running_instances == 0
|
|
return success, running_instances
|
|
|
|
def _check_simplex_available_space(self):
|
|
"""Ensures there is free space for the backup"""
|
|
try:
|
|
backup_restore.check_size("/opt/backups", True)
|
|
except backup_restore.BackupFail:
|
|
return False
|
|
|
|
return True
|
|
|
|
def get_system_health(self, context, force=False):
|
|
"""Returns the general health of the system"""
|
|
# Checks the following:
|
|
# All hosts are provisioned
|
|
# All hosts are patch current
|
|
# All hosts are unlocked/enabled
|
|
# All hosts having matching configs
|
|
# No management affecting alarms
|
|
# For ceph systems: The storage cluster is healthy
|
|
|
|
hosts = self._dbapi.ihost_get_list()
|
|
output = _('System Health:\n')
|
|
health_ok = True
|
|
|
|
unprovisioned_hosts, provisioned_hosts = \
|
|
self._check_hosts_provisioned(hosts)
|
|
success = unprovisioned_hosts == 0
|
|
output += (_('All hosts are provisioned: [%s]\n')
|
|
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG))
|
|
if not success:
|
|
output += _('%s Unprovisioned hosts\n') % unprovisioned_hosts
|
|
# Set the hosts to the provisioned_hosts. This will allow the other
|
|
# checks to continue
|
|
hosts = provisioned_hosts
|
|
|
|
health_ok = health_ok and success
|
|
|
|
success, error_hosts = self._check_hosts_enabled(hosts)
|
|
output += _('All hosts are unlocked/enabled: [%s]\n') \
|
|
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
|
|
if not success:
|
|
output += _('Locked or disabled hosts: %s\n') \
|
|
% ', '.join(error_hosts)
|
|
|
|
health_ok = health_ok and success
|
|
|
|
success, error_hosts = self._check_hosts_config(hosts)
|
|
output += _('All hosts have current configurations: [%s]\n') \
|
|
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
|
|
if not success:
|
|
output += _('Hosts with out of date configurations: %s\n') \
|
|
% ', '.join(error_hosts)
|
|
|
|
health_ok = health_ok and success
|
|
|
|
success, error_hosts, missing_hosts = self._check_patch_current(hosts)
|
|
output += _('All hosts are patch current: [%s]\n') \
|
|
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
|
|
if not success:
|
|
if error_hosts:
|
|
output += _('Hosts not patch current: %s\n') \
|
|
% ', '.join(error_hosts)
|
|
if missing_hosts:
|
|
output += _('Hosts without patch data: %s\n') \
|
|
% ', '.join(missing_hosts)
|
|
|
|
health_ok = health_ok and success
|
|
|
|
if StorageBackendConfig.has_backend(
|
|
self._dbapi,
|
|
constants.CINDER_BACKEND_CEPH):
|
|
success = self._check_ceph()
|
|
output += _('Ceph Storage Healthy: [%s]\n') \
|
|
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
|
|
|
|
health_ok = health_ok and success
|
|
|
|
success, allowed, affecting = self._check_alarms(context, force)
|
|
output += _('No alarms: [%s]\n') \
|
|
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
|
|
if not success:
|
|
output += _('[%s] alarms found, [%s] of which are management '
|
|
'affecting\n') % (allowed + affecting, affecting)
|
|
|
|
health_ok = health_ok and success
|
|
|
|
return health_ok, output
|
|
|
|
def get_system_health_upgrade(self, context, force=False):
|
|
"""Ensures the system is in a valid state for an upgrade"""
|
|
# Does a general health check then does the following:
|
|
# A load is imported
|
|
# The load patch requirements are met
|
|
# The license is valid for the N+1 load
|
|
|
|
system_mode = self._dbapi.isystem_get_one().system_mode
|
|
simplex = (system_mode == constants.SYSTEM_MODE_SIMPLEX)
|
|
|
|
health_ok, output = self.get_system_health(context, force)
|
|
loads = self._dbapi.load_get_list()
|
|
try:
|
|
imported_load = utils.get_imported_load(loads)
|
|
except Exception as e:
|
|
LOG.exception(e)
|
|
output += _('No imported load found. Unable to test further\n')
|
|
return health_ok, output
|
|
|
|
upgrade_version = imported_load.software_version
|
|
if imported_load.required_patches:
|
|
patches = imported_load.required_patches.split('\n')
|
|
else:
|
|
patches = []
|
|
|
|
success, missing_patches = self._check_required_patches(patches)
|
|
output += _('Required patches are applied: [%s]\n') \
|
|
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
|
|
if not success:
|
|
output += _('Patches not applied: %s\n') \
|
|
% ', '.join(missing_patches)
|
|
|
|
health_ok = health_ok and success
|
|
|
|
success = self._check_license(upgrade_version)
|
|
output += _('License valid for upgrade: [%s]\n') \
|
|
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
|
|
|
|
health_ok = health_ok and success
|
|
|
|
if not simplex:
|
|
controller_1 = self._dbapi.ihost_get_by_hostname(
|
|
constants.CONTROLLER_1_HOSTNAME)
|
|
|
|
# If we are running on CPE we don't want any instances running
|
|
# on controller-1 before we start the upgrade, otherwise the
|
|
# databases will be out of sync after we lock controller-1
|
|
if constants.WORKER in controller_1.subfunctions:
|
|
success, running_instances = self._check_running_instances(
|
|
controller_1)
|
|
output += \
|
|
_('No instances running on controller-1: [%s]\n') \
|
|
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
|
|
if not success:
|
|
output += _('Number of instances on controller-1: %s\n') \
|
|
% (running_instances)
|
|
|
|
health_ok = health_ok and success
|
|
else:
|
|
success = self._check_simplex_available_space()
|
|
output += \
|
|
_('Sufficient free space for upgrade: [%s]\n') \
|
|
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
|
|
|
|
health_ok = health_ok and success
|
|
|
|
return health_ok, output
|