5030 lines
185 KiB
Python
Executable File
5030 lines
185 KiB
Python
Executable File
#
|
|
# Copyright (c) 2015-2024 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
import six
|
|
|
|
from nfv_common import debug
|
|
from nfv_common.helpers import Constant
|
|
from nfv_common.helpers import Constants
|
|
from nfv_common.helpers import coroutine
|
|
from nfv_common.helpers import Singleton
|
|
from nfv_common import strategy
|
|
from nfv_common import timers
|
|
from nfv_vim import objects
|
|
from nfv_vim.strategy._strategy_defs import FW_UPDATE_LABEL
|
|
from nfv_vim.strategy._strategy_defs import STRATEGY_EVENT
|
|
from nfv_vim import tables
|
|
|
|
DLOG = debug.debug_get_logger('nfv_vim.strategy.step')
|
|
|
|
# todo(abailey): create a kube rootca update constants file
|
|
KUBE_CERT_UPDATE_TRUSTBOTHCAS = "trust-both-cas"
|
|
KUBE_CERT_UPDATE_TRUSTNEWCA = "trust-new-ca"
|
|
KUBE_CERT_UPDATE_UPDATECERTS = "update-certs"
|
|
|
|
|
|
@six.add_metaclass(Singleton)
|
|
class StrategyStepNames(Constants):
|
|
"""
|
|
Strategy Step Names
|
|
"""
|
|
QUERY_HOSTS = Constant('query-hosts')
|
|
SYSTEM_STABILIZE = Constant('system-stabilize')
|
|
LOCK_HOSTS = Constant('lock-hosts')
|
|
UNLOCK_HOSTS = Constant('unlock-hosts')
|
|
REBOOT_HOSTS = Constant('reboot-hosts')
|
|
UPGRADE_HOSTS = Constant('upgrade-hosts')
|
|
START_UPGRADE = Constant('start-upgrade')
|
|
ACTIVATE_UPGRADE = Constant('activate-upgrade')
|
|
COMPLETE_UPGRADE = Constant('complete-upgrade')
|
|
SWACT_HOSTS = Constant('swact-hosts')
|
|
SW_PATCH_HOSTS = Constant('sw-patch-hosts')
|
|
FW_UPDATE_HOSTS = Constant('fw-update-hosts')
|
|
FW_UPDATE_ABORT_HOSTS = Constant('fw-update-abort-hosts')
|
|
MIGRATE_INSTANCES = Constant('migrate-instances')
|
|
MIGRATE_INSTANCES_FROM_HOST = Constant('migrate-instances-from-host')
|
|
STOP_INSTANCES = Constant('stop-instances')
|
|
START_INSTANCES = Constant('start-instances')
|
|
QUERY_ALARMS = Constant('query-alarms')
|
|
WAIT_DATA_SYNC = Constant('wait-data-sync')
|
|
WAIT_ALARMS_CLEAR = Constant('wait-alarms-clear')
|
|
QUERY_SW_PATCHES = Constant('query-sw-patches')
|
|
QUERY_SW_PATCH_HOSTS = Constant('query-sw-patch-hosts')
|
|
QUERY_FW_UPDATE_HOST = Constant('query-fw-update-host')
|
|
QUERY_UPGRADE = Constant('query-upgrade')
|
|
DISABLE_HOST_SERVICES = Constant('disable-host-services')
|
|
ENABLE_HOST_SERVICES = Constant('enable-host-services')
|
|
# kube rootca update steps
|
|
KUBE_ROOTCA_UPDATE_ABORT = Constant('kube-rootca-update-abort')
|
|
KUBE_ROOTCA_UPDATE_COMPLETE = Constant('kube-rootca-update-complete')
|
|
KUBE_ROOTCA_UPDATE_GENERATE_CERT = Constant('kube-rootca-update-generate-cert')
|
|
KUBE_ROOTCA_UPDATE_HOST_TRUSTBOTHCAS = Constant('kube-rootca-update-host-trustbothcas')
|
|
KUBE_ROOTCA_UPDATE_HOST_TRUSTNEWCA = Constant('kube-rootca-update-host-trustnewca')
|
|
KUBE_ROOTCA_UPDATE_HOST_UPDATECERTS = Constant('kube-rootca-update-host-update-certs')
|
|
KUBE_ROOTCA_UPDATE_PODS_TRUSTBOTHCAS = Constant('kube-rootca-update-pods-trustbothcas')
|
|
KUBE_ROOTCA_UPDATE_PODS_TRUSTNEWCA = Constant('kube-rootca-update-pods-trustnewca')
|
|
KUBE_ROOTCA_UPDATE_START = Constant('kube-rootca-update-start')
|
|
KUBE_ROOTCA_UPDATE_UPLOAD_CERT = Constant('kube-rootca-update-upload-cert')
|
|
QUERY_KUBE_ROOTCA_UPDATE = Constant('query-kube-rootca-update')
|
|
QUERY_KUBE_ROOTCA_HOST_UPDATES = Constant('query-kube-rootca-host-updates')
|
|
# kube upgrade steps
|
|
APPLY_PATCHES = Constant('apply-patches')
|
|
QUERY_KUBE_HOST_UPGRADE = Constant('query-kube-host-upgrade')
|
|
QUERY_KUBE_UPGRADE = Constant('query-kube-upgrade')
|
|
QUERY_KUBE_VERSIONS = Constant('query-kube-versions')
|
|
KUBE_HOST_CORDON = Constant('kube-host-cordon')
|
|
KUBE_HOST_UNCORDON = Constant('kube-host-uncordon')
|
|
KUBE_UPGRADE_ABORT = Constant('kube-upgrade-abort')
|
|
KUBE_UPGRADE_START = Constant('kube-upgrade-start')
|
|
KUBE_UPGRADE_CLEANUP = Constant('kube-upgrade-cleanup')
|
|
KUBE_UPGRADE_CLEANUP_ABORTED = Constant('kube-upgrade-cleanup-aborted')
|
|
KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete')
|
|
KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images')
|
|
KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking')
|
|
KUBE_UPGRADE_STORAGE = Constant('kube-upgrade-storage')
|
|
KUBE_HOST_UPGRADE_CONTROL_PLANE = \
|
|
Constant('kube-host-upgrade-control-plane')
|
|
KUBE_HOST_UPGRADE_KUBELET = Constant('kube-host-upgrade-kubelet')
|
|
# system config update specific steps
|
|
QUERY_SYSTEM_CONFIG_UPDATE_HOSTS = Constant('query-system-config-update-hosts')
|
|
SYSTEM_CONFIG_UPDATE_HOSTS = Constant('system-config-update-hosts')
|
|
|
|
|
|
# Constant Instantiation
|
|
STRATEGY_STEP_NAME = StrategyStepNames()
|
|
|
|
|
|
def validate_operation(operation):
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
|
|
class AbstractStrategyStep(strategy.StrategyStep):
|
|
"""An abstract base class for strategy steps"""
|
|
|
|
def __init__(self, step_name, timeout_in_secs):
|
|
super(AbstractStrategyStep, self).__init__(
|
|
step_name,
|
|
timeout_in_secs=timeout_in_secs)
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(AbstractStrategyStep, self).from_dict(data)
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the step as a dictionary
|
|
"""
|
|
data = super(AbstractStrategyStep, self).as_dict()
|
|
# Next 3 lines are required for all strategy steps and may be
|
|
# overridden by subclass in some cases
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = list()
|
|
data['entity_uuids'] = list()
|
|
return data
|
|
|
|
|
|
class AbstractHostsStrategyStep(AbstractStrategyStep):
|
|
"""An abstract base class for strategy steps performed on list of hosts"""
|
|
|
|
def __init__(self,
|
|
step_name,
|
|
hosts,
|
|
timeout_in_secs=1800):
|
|
super(AbstractHostsStrategyStep, self).__init__(
|
|
step_name,
|
|
timeout_in_secs=timeout_in_secs)
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(AbstractHostsStrategyStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the step as a dictionary
|
|
"""
|
|
data = super(AbstractHostsStrategyStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
return data
|
|
|
|
|
|
class UnlockHostsStep(AbstractHostsStrategyStep):
|
|
"""
|
|
Unlock Hosts - Strategy Step
|
|
"""
|
|
|
|
# During an upgrade, an unlock may need to be retried several times
|
|
# https://bugs.launchpad.net/starlingx/+bug/1914836
|
|
MAX_RETRIES = 5
|
|
RETRY_DELAY = 120
|
|
|
|
def __init__(self, hosts, retry_count=0, retry_delay=RETRY_DELAY):
|
|
"""
|
|
hosts - the list of hosts to be unlocked
|
|
retry_count - the number of times to retry per host if unlock fails
|
|
retry_delay - the amount of time to delay before retrying unlock
|
|
"""
|
|
super(UnlockHostsStep, self).__init__(STRATEGY_STEP_NAME.UNLOCK_HOSTS,
|
|
hosts,
|
|
timeout_in_secs=1800)
|
|
# step_name, hosts, timeout are serialized by parent classes
|
|
# retry_count and retry_delay must be serialized in from_dict/as_dict
|
|
self._retry_count = retry_count
|
|
self._retry_delay = retry_delay
|
|
# Do not persist: _retries, _wait_time _retrying
|
|
self._retries = dict()
|
|
for host_name in self._host_names:
|
|
self._retries[host_name] = retry_count
|
|
self._wait_time = 0
|
|
self._retry_requested = False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns unlock hosts step object initialized using the given dictionary
|
|
"""
|
|
super(UnlockHostsStep, self).from_dict(data)
|
|
# deserialize retry_delay and retry_count
|
|
# 'retry_delay' and 'retry_count' were added to this step since last
|
|
# release. Need to perform 'get' with a default value in case
|
|
# we are deserializing a strategy that does not contain these keys.
|
|
self._retry_count = data.get('retry_count', 0)
|
|
self._retry_delay = data.get('retry_delay', self.RETRY_DELAY)
|
|
|
|
# Do not deserialize _retries, _wait_time and _retrying
|
|
self._wait_time = 0
|
|
self._retry_requested = False
|
|
self._retries = dict()
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._retries[host_name] = self._retry_count
|
|
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the unlock hosts step as a dictionary
|
|
"""
|
|
data = super(UnlockHostsStep, self).as_dict()
|
|
# serialize retries
|
|
data['retry_count'] = self._retry_count
|
|
# serialize retry_delay
|
|
data['retry_delay'] = self._retry_delay
|
|
# Do not serialize _retries, _wait_time and _retrying
|
|
return data
|
|
|
|
def _get_hosts_to_retry(self):
|
|
hosts = []
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is None:
|
|
continue
|
|
if host.is_locked() and self._retries[host_name] > 0:
|
|
self._retries[host_name] = self._retries[host_name] - 1
|
|
hosts.append(host_name)
|
|
return hosts
|
|
|
|
def _total_hosts_unlocked_enabled(self):
|
|
"""
|
|
Returns the number of hosts that are unlocked and enabled
|
|
"""
|
|
total_hosts_enabled = 0
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is None:
|
|
return -1
|
|
|
|
if not host.is_locked() and host.is_enabled():
|
|
total_hosts_enabled += 1
|
|
|
|
return total_hosts_enabled
|
|
|
|
def _trigger_retry(self, host_name):
|
|
DLOG.info("Step (%s) retry due to failure for (%s)." % (self._name,
|
|
host_name))
|
|
# set the retry trigger
|
|
self._retry_requested = True
|
|
# reset the retry "wait" delay
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
# decrement the number of allowed retries for the validated host
|
|
self._retries[host_name] = self._retries[host_name] - 1
|
|
|
|
def apply(self):
|
|
"""
|
|
Unlock all hosts
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
|
|
self._host_names))
|
|
if len(self._host_names) == self._total_hosts_unlocked_enabled():
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
host_director = directors.get_host_director()
|
|
operation = host_director.unlock_hosts(self._host_names)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event in [STRATEGY_EVENT.HOST_STATE_CHANGED,
|
|
STRATEGY_EVENT.HOST_AUDIT]:
|
|
total_hosts_enabled = self._total_hosts_unlocked_enabled()
|
|
|
|
if -1 == total_hosts_enabled:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host no longer exists")
|
|
return True
|
|
|
|
if total_hosts_enabled == len(self._host_names):
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
# See if we have requested a retry and are not currently retrying
|
|
if self._retry_requested:
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
if self._retry_delay <= secs_expired:
|
|
self._retry_requested = False
|
|
# re-issue unlock for all hosts.
|
|
# Hosts that are already unlocked or unlocking get skipped
|
|
host_director = directors.get_host_director()
|
|
operation = host_director.unlock_hosts(self._host_names)
|
|
if operation.is_failed():
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host unlock failed")
|
|
return True
|
|
|
|
elif event == STRATEGY_EVENT.HOST_UNLOCK_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
if host.is_locked() and self._retries[host.name] > 0:
|
|
# if any unlock fails and we have retries, trigger it
|
|
# even if the last round of unlocks has not returned
|
|
self._trigger_retry(host.name)
|
|
else:
|
|
# if ANY unlock fails and we are out of retries, fail
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host unlock failed")
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
class LockHostsStep(strategy.StrategyStep):
|
|
"""
|
|
Lock Hosts - Strategy Step
|
|
"""
|
|
def __init__(self, hosts, wait_until_disabled=True):
|
|
super(LockHostsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.LOCK_HOSTS, timeout_in_secs=900)
|
|
self._hosts = hosts
|
|
self._wait_until_disabled = wait_until_disabled
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
self._wait_time = 0
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
return [UnlockHostsStep(self._hosts)]
|
|
|
|
def _total_hosts_locked(self):
|
|
"""
|
|
Returns the number of hosts that are locked
|
|
"""
|
|
total_hosts_locked = 0
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is None:
|
|
return -1
|
|
|
|
if host.is_locked() and \
|
|
(not self._wait_until_disabled or host.is_disabled()):
|
|
total_hosts_locked += 1
|
|
|
|
return total_hosts_locked
|
|
|
|
def _instances_not_locked_disabled(self):
|
|
"""
|
|
Returns the instances that are not locked and disabled
|
|
"""
|
|
instances_not_locked_disabled = []
|
|
instance_table = tables.tables_get_instance_table()
|
|
for host_name in self._host_names:
|
|
for instance in instance_table.on_host(host_name):
|
|
if not (instance.is_locked() and instance.is_disabled()):
|
|
instances_not_locked_disabled.append(instance.name)
|
|
|
|
return instances_not_locked_disabled
|
|
|
|
def apply(self):
|
|
"""
|
|
Lock all hosts
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
|
|
self._host_names))
|
|
if len(self._host_names) == self._total_hosts_locked():
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
# Ensure that no instances are running on these hosts before locking.
|
|
# Instances should have been stopped or migrated to another host by now.
|
|
instances = self._instances_not_locked_disabled()
|
|
if instances:
|
|
reason = ("Lock of host(s) %s failed because instance(s) %s were "
|
|
"not migrated or stopped." % (','.join(self._host_names),
|
|
','.join(instances)))
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, reason
|
|
|
|
host_director = directors.get_host_director()
|
|
operation = host_director.lock_hosts(self._host_names)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event in [STRATEGY_EVENT.HOST_STATE_CHANGED, STRATEGY_EVENT.HOST_AUDIT]:
|
|
total_hosts_locked = self._total_hosts_locked()
|
|
|
|
if -1 == total_hosts_locked:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host no longer exists")
|
|
return True
|
|
|
|
if not self._wait_until_disabled:
|
|
# If we are not waiting for the hosts to go disabled, then wait
|
|
# at least 15 seconds after doing the lock.
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
if 15 >= secs_expired:
|
|
return True
|
|
|
|
if total_hosts_locked == len(self._host_names):
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
elif event == STRATEGY_EVENT.HOST_LOCK_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host lock failed")
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the lock hosts step object initialized using the given dictionary
|
|
"""
|
|
super(LockHostsStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._wait_until_disabled = data['wait_until_disabled']
|
|
self._host_uuids = list()
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
self._wait_time = 0
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the lock hosts step as a dictionary
|
|
"""
|
|
data = super(LockHostsStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
data['wait_until_disabled'] = self._wait_until_disabled
|
|
return data
|
|
|
|
|
|
class RebootHostsStep(strategy.StrategyStep):
|
|
"""
|
|
Reboot Hosts - Strategy Step
|
|
"""
|
|
def __init__(self, hosts):
|
|
super(RebootHostsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.REBOOT_HOSTS, timeout_in_secs=900)
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
self._wait_time = 0
|
|
|
|
def apply(self):
|
|
"""
|
|
Reboot all hosts
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
|
|
self._host_names))
|
|
host_director = directors.get_host_director()
|
|
operation = host_director.reboot_hosts(self._host_names)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_REBOOT_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host reboot failed")
|
|
return True
|
|
|
|
elif event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
if 60 <= secs_expired:
|
|
# Wait 60 seconds, which should be enough time for the host
|
|
# to shutdown and reboot. No need to wait for the host to
|
|
# come back online since it will remain locked after the reboot.
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the reboot hosts step object initialized using the given dictionary
|
|
"""
|
|
super(RebootHostsStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
self._wait_time = 0
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the reboot hosts step as a dictionary
|
|
"""
|
|
data = super(RebootHostsStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
return data
|
|
|
|
|
|
class SwactHostsStep(strategy.StrategyStep):
|
|
"""
|
|
Swact Hosts - Strategy Step
|
|
"""
|
|
def __init__(self, hosts):
|
|
super(SwactHostsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.SWACT_HOSTS, timeout_in_secs=900)
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
self._wait_time = 0
|
|
|
|
def apply(self):
|
|
"""
|
|
Swact hosts
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
|
|
self._host_names))
|
|
host_director = directors.get_host_director()
|
|
operation = host_director.swact_hosts(self._host_names)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_SWACT_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host swact failed")
|
|
return True
|
|
|
|
elif event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
if 120 <= secs_expired:
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the swact hosts step object initialized using the given dictionary
|
|
"""
|
|
super(SwactHostsStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
self._wait_time = 0
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the swact hosts step as a dictionary
|
|
"""
|
|
data = super(SwactHostsStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
return data
|
|
|
|
|
|
class SwPatchHostsStep(strategy.StrategyStep):
|
|
"""
|
|
Software Patch Hosts - Strategy Step
|
|
"""
|
|
def __init__(self, hosts):
|
|
super(SwPatchHostsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.SW_PATCH_HOSTS, timeout_in_secs=1800)
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
self._host_completed = dict()
|
|
self._query_inprogress = False
|
|
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
self._host_completed[host.name] = (False, False, '')
|
|
|
|
@coroutine
|
|
def _query_hosts_callback(self):
|
|
"""
|
|
Query Software Patch Hosts Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Query-Hosts callback response=%s." % response)
|
|
|
|
self._query_inprogress = False
|
|
|
|
if response['completed']:
|
|
for sw_patch_host in response['result-data']:
|
|
if self._host_completed.get(sw_patch_host.name, False):
|
|
# A patch can be listed as failed, and patch current
|
|
# so check the failed state first.
|
|
if sw_patch_host.patch_failed:
|
|
self._host_completed[sw_patch_host.name] = \
|
|
(True, False, "software update failed to apply on "
|
|
"host %s" % sw_patch_host.name)
|
|
elif sw_patch_host.patch_current:
|
|
self._host_completed[sw_patch_host.name] = \
|
|
(True, True, '')
|
|
|
|
failed = False
|
|
failed_reason = ''
|
|
|
|
for host_name in self._host_completed:
|
|
completed, success, reason = self._host_completed[host_name]
|
|
if not completed:
|
|
break
|
|
|
|
if not success:
|
|
failed = True
|
|
failed_reason = reason
|
|
|
|
else:
|
|
if failed:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, failed_reason)
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
|
|
@coroutine
|
|
def _sw_patch_hosts_callback(self):
|
|
"""
|
|
Software Patch Hosts Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Software-Update-Hosts callback response=%s." % response)
|
|
|
|
if not response['completed']:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
Software Patch Hosts
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
|
|
self._host_names))
|
|
nfvi.nfvi_sw_mgmt_update_hosts(self._host_names,
|
|
self._sw_patch_hosts_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if not self._query_inprogress:
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_sw_mgmt_query_hosts(self._query_hosts_callback())
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the software patch hosts step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(SwPatchHostsStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_completed = dict()
|
|
self._query_inprogress = False
|
|
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
self._host_completed[host_name] = \
|
|
data['hosts_completed'][host_name]
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the software patch hosts step as a dictionary
|
|
"""
|
|
data = super(SwPatchHostsStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
data['hosts_completed'] = self._host_completed
|
|
return data
|
|
|
|
|
|
class SystemConfigUpdateHostsStep(strategy.StrategyStep):
|
|
"""
|
|
System Config Update Hosts - Strategy Step
|
|
"""
|
|
def __init__(self, hosts):
|
|
super(SystemConfigUpdateHostsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.SYSTEM_CONFIG_UPDATE_HOSTS, timeout_in_secs=1800)
|
|
self._hosts = hosts
|
|
self._wait_time = 0
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
self._query_inprogress = False
|
|
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
|
|
@coroutine
|
|
def _query_hosts_callback(self):
|
|
"""
|
|
Query System Config Update Hosts Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Query-Hosts callback response=%s." % response)
|
|
|
|
self._query_inprogress = False
|
|
|
|
if response['completed']:
|
|
for host in response['result-data']:
|
|
if host.unlock_request != 'unlock_required':
|
|
# Keep waiting for the hosts to reach the unlock required
|
|
# status
|
|
pass
|
|
|
|
# Ready to unlock the hosts
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
System Config Update Hosts
|
|
"""
|
|
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
|
|
self._host_names))
|
|
# Do nothing, wait for the callback to check the unlock request status
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
# Wait at least 30 seconds before checking the unlock status for
|
|
# first time
|
|
if 30 <= secs_expired and not self._query_inprogress:
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_system_config_unlock_request(
|
|
self._host_names, self._query_hosts_callback())
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the system config update hosts step object initialized using
|
|
the given dictionary
|
|
"""
|
|
super(SystemConfigUpdateHostsStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._wait_time = 0
|
|
self._host_uuids = list()
|
|
self._query_inprogress = False
|
|
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the system config update hosts step as a dictionary
|
|
"""
|
|
data = super(SystemConfigUpdateHostsStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
return data
|
|
|
|
|
|
class UpgradeHostsStep(strategy.StrategyStep):
|
|
"""
|
|
Upgrade Hosts - Strategy Step
|
|
"""
|
|
def __init__(self, hosts):
|
|
super(UpgradeHostsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.UPGRADE_HOSTS, timeout_in_secs=3600)
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
self._wait_time = 0
|
|
|
|
def _total_hosts_upgraded(self):
|
|
"""
|
|
Returns the number of hosts that are upgraded
|
|
"""
|
|
total_hosts_upgraded = 0
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is None:
|
|
return -1
|
|
|
|
if (host.is_online() and
|
|
host.target_load == self.strategy.nfvi_upgrade.to_release and
|
|
host.software_load == self.strategy.nfvi_upgrade.to_release):
|
|
total_hosts_upgraded += 1
|
|
|
|
return total_hosts_upgraded
|
|
|
|
def apply(self):
|
|
"""
|
|
Upgrade all hosts
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
|
|
self._host_names))
|
|
host_director = directors.get_host_director()
|
|
operation = host_director.upgrade_hosts(self._host_names)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_UPGRADE_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host upgrade failed")
|
|
return True
|
|
|
|
elif event in [STRATEGY_EVENT.HOST_AUDIT]:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
# Wait at least 2 minutes for the host to go offline before
|
|
# checking whether the upgrade is complete.
|
|
if 120 <= secs_expired:
|
|
total_hosts_upgraded = self._total_hosts_upgraded()
|
|
|
|
if -1 == total_hosts_upgraded:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host no longer exists")
|
|
return True
|
|
|
|
if total_hosts_upgraded == len(self._host_names):
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the upgrade hosts step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(UpgradeHostsStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
self._wait_time = 0
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the upgrade hosts step as a dictionary
|
|
"""
|
|
data = super(UpgradeHostsStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
return data
|
|
|
|
|
|
class UpgradeStartStep(strategy.StrategyStep):
|
|
"""
|
|
Upgrade Start - Strategy Step
|
|
"""
|
|
def __init__(self):
|
|
super(UpgradeStartStep, self).__init__(
|
|
STRATEGY_STEP_NAME.START_UPGRADE, timeout_in_secs=600)
|
|
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
|
|
@coroutine
|
|
def _start_upgrade_callback(self):
|
|
"""
|
|
Start Upgrade Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Start-Upgrade callback response=%s." % response)
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_upgrade = response['result-data']
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
@coroutine
|
|
def _get_upgrade_callback(self):
|
|
"""
|
|
Get Upgrade Callback
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
response = (yield)
|
|
DLOG.debug("Get-Upgrade callback response=%s." % response)
|
|
|
|
self._query_inprogress = False
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_upgrade = response['result-data']
|
|
|
|
if self.strategy.nfvi_upgrade.state != \
|
|
nfvi.objects.v1.UPGRADE_STATE.STARTED:
|
|
# Keep waiting for upgrade to start
|
|
pass
|
|
else:
|
|
# Upgrade has started
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
Upgrade Start
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_upgrade_start(self._start_upgrade_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
# Wait at least 60 seconds before checking upgrade for first time
|
|
if 60 <= secs_expired and not self._query_inprogress:
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the upgrade start step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(UpgradeStartStep, self).from_dict(data)
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the upgrade start step as a dictionary
|
|
"""
|
|
data = super(UpgradeStartStep, self).as_dict()
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = list()
|
|
data['entity_uuids'] = list()
|
|
return data
|
|
|
|
|
|
class UpgradeActivateStep(strategy.StrategyStep):
|
|
"""
|
|
Upgrade Activate - Strategy Step
|
|
"""
|
|
|
|
def __init__(self):
|
|
super(UpgradeActivateStep, self).__init__(
|
|
STRATEGY_STEP_NAME.ACTIVATE_UPGRADE, timeout_in_secs=900)
|
|
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
|
|
@coroutine
|
|
def _activate_upgrade_callback(self):
|
|
"""
|
|
Activate Upgrade Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Activate-Upgrade callback response=%s." % response)
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_upgrade = response['result-data']
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
@coroutine
|
|
def _get_upgrade_callback(self):
|
|
"""
|
|
Get Upgrade Callback
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
response = (yield)
|
|
DLOG.debug("Get-Upgrade callback response=%s." % response)
|
|
|
|
self._query_inprogress = False
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_upgrade = response['result-data']
|
|
|
|
if self.strategy.nfvi_upgrade.state != \
|
|
nfvi.objects.v1.UPGRADE_STATE.ACTIVATION_COMPLETE:
|
|
# Keep waiting for upgrade to activate
|
|
pass
|
|
else:
|
|
# Upgrade has activated
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
Upgrade Activate
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_upgrade_activate(self._activate_upgrade_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
# Wait at least 60 seconds before checking upgrade for first time
|
|
if 60 <= secs_expired and not self._query_inprogress:
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the upgrade activate step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(UpgradeActivateStep, self).from_dict(data)
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the upgrade activate step as a dictionary
|
|
"""
|
|
data = super(UpgradeActivateStep, self).as_dict()
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = list()
|
|
data['entity_uuids'] = list()
|
|
return data
|
|
|
|
|
|
class UpgradeCompleteStep(strategy.StrategyStep):
|
|
"""
|
|
Upgrade Complete - Strategy Step
|
|
"""
|
|
|
|
def __init__(self):
|
|
super(UpgradeCompleteStep, self).__init__(
|
|
STRATEGY_STEP_NAME.COMPLETE_UPGRADE, timeout_in_secs=300)
|
|
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
|
|
@coroutine
|
|
def _complete_upgrade_callback(self):
|
|
"""
|
|
Complete Upgrade Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Complete-Upgrade callback response=%s." % response)
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_upgrade = response['result-data']
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
@coroutine
|
|
def _get_upgrade_callback(self):
|
|
"""
|
|
Get Upgrade Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Get-Upgrade callback response=%s." % response)
|
|
|
|
self._query_inprogress = False
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_upgrade = response['result-data']
|
|
|
|
if self.strategy.nfvi_upgrade is not None:
|
|
# Keep waiting for upgrade to complete
|
|
pass
|
|
else:
|
|
# Upgrade has been completed (upgrade record deleted)
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
Upgrade Complete
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_upgrade_complete(self._complete_upgrade_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
# Wait at least 60 seconds before checking upgrade for first time
|
|
if 60 <= secs_expired and not self._query_inprogress:
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the upgrade complete step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(UpgradeCompleteStep, self).from_dict(data)
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the upgrade complete step as a dictionary
|
|
"""
|
|
data = super(UpgradeCompleteStep, self).as_dict()
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = list()
|
|
data['entity_uuids'] = list()
|
|
return data
|
|
|
|
|
|
class MigrateInstancesFromHostStep(strategy.StrategyStep):
|
|
"""
|
|
Migrate Instances From Host - Strategy Step
|
|
"""
|
|
def __init__(self, hosts, instances):
|
|
super(MigrateInstancesFromHostStep, self).__init__(
|
|
STRATEGY_STEP_NAME.MIGRATE_INSTANCES_FROM_HOST, timeout_in_secs=1800)
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._instances = instances
|
|
self._instance_names = list()
|
|
self._instance_uuids = list()
|
|
self._instance_host_names = dict()
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
for instance in instances:
|
|
self._instance_names.append(instance.name)
|
|
self._instance_uuids.append(instance.uuid)
|
|
self._instance_host_names[instance.uuid] = instance.host_name
|
|
|
|
def _all_instances_migrated(self):
|
|
"""
|
|
Returns true if all instances have migrated from the source hosts
|
|
"""
|
|
instance_table = tables.tables_get_instance_table()
|
|
for host_name in self._host_names:
|
|
if instance_table.exist_on_host(host_name):
|
|
return False, ""
|
|
|
|
return True, ""
|
|
|
|
def apply(self):
|
|
"""
|
|
Migrate all instances
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
if(self._instance_names):
|
|
DLOG.info("Step (%s) apply for instances %s running on hosts %s." % (
|
|
self._name,
|
|
self._instance_names,
|
|
self._host_names,
|
|
))
|
|
else:
|
|
DLOG.info("Step (%s) apply in no instances on empty hosts %s." % (
|
|
self._name,
|
|
self._host_names,
|
|
))
|
|
|
|
migrate_complete, reason = self._all_instances_migrated()
|
|
if migrate_complete:
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
# Ensure none of the instances have moved since the strategy step was
|
|
# created. The instance_director.migrate_instances will migrate ALL
|
|
# instances on each host containing one of the self._instance_uuids. We
|
|
# want to ensure we are only migrating instances from the host(s) they
|
|
# were originally located on..
|
|
for instance in self._instances:
|
|
if instance.host_name != self._instance_host_names[instance.uuid]:
|
|
reason = ("instance %s has moved from %s to %s after strategy "
|
|
"created" %
|
|
(instance.name, self._instance_host_names[instance.uuid],
|
|
instance.host_name))
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, reason
|
|
|
|
instance_director = directors.get_instance_director()
|
|
operation = instance_director.migrate_instances_from_hosts(self._host_names)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Instance events
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event in [STRATEGY_EVENT.INSTANCE_STATE_CHANGED,
|
|
STRATEGY_EVENT.INSTANCE_AUDIT,
|
|
STRATEGY_EVENT.HOST_AUDIT]:
|
|
migrate_complete, reason = self._all_instances_migrated()
|
|
|
|
if not migrate_complete and reason:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, reason)
|
|
return True
|
|
|
|
if migrate_complete:
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
elif STRATEGY_EVENT.MIGRATE_INSTANCES_FAILED == event:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, event_data)
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the migrate instances from hosts step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(MigrateInstancesFromHostStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._host_names = data['host_names']
|
|
self._instance_uuids = data['entity_uuids']
|
|
self._instances = list()
|
|
self._instance_names = list()
|
|
self._instance_host_names = dict()
|
|
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host:
|
|
self._hosts.append(host)
|
|
|
|
instance_table = tables.tables_get_instance_table()
|
|
for instance_uuid in self._instance_uuids:
|
|
instance = instance_table.get(instance_uuid, None)
|
|
if instance is not None:
|
|
self._instances.append(instance)
|
|
self._instance_names.append(instance.name)
|
|
# Retrieve the host this instance was on when the step was
|
|
# created.
|
|
self._instance_host_names[instance.uuid] = \
|
|
data['instance_host_names'][instance.uuid]
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the migrate instances from hosts step as a dictionary
|
|
"""
|
|
data = super(MigrateInstancesFromHostStep, self).as_dict()
|
|
data['entity_type'] = 'instances'
|
|
data['entity_names'] = self._instance_names
|
|
data['entity_uuids'] = self._instance_uuids
|
|
data['instance_host_names'] = self._instance_host_names
|
|
data['host_names'] = self._host_names
|
|
return data
|
|
|
|
|
|
class MigrateInstancesStep(strategy.StrategyStep):
|
|
"""
|
|
Migrate Instances - Strategy Step
|
|
"""
|
|
def __init__(self, instances):
|
|
super(MigrateInstancesStep, self).__init__(
|
|
STRATEGY_STEP_NAME.MIGRATE_INSTANCES, timeout_in_secs=1800)
|
|
self._instances = instances
|
|
self._instance_names = list()
|
|
self._instance_uuids = list()
|
|
self._instance_host_names = dict()
|
|
for instance in instances:
|
|
self._instance_names.append(instance.name)
|
|
self._instance_uuids.append(instance.uuid)
|
|
self._instance_host_names[instance.uuid] = instance.host_name
|
|
|
|
def _all_instances_migrated(self):
|
|
"""
|
|
Returns true if all instances have migrated from the source hosts
|
|
"""
|
|
source_host_names = []
|
|
for host_name in list(self._instance_host_names.values()):
|
|
if host_name not in source_host_names:
|
|
source_host_names.append(host_name)
|
|
|
|
instance_table = tables.tables_get_instance_table()
|
|
for host_name in source_host_names:
|
|
if instance_table.exist_on_host(host_name):
|
|
return False, ""
|
|
|
|
return True, ""
|
|
|
|
def apply(self):
|
|
"""
|
|
Migrate all instances
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for instances %s." % (self._name,
|
|
self._instance_names))
|
|
migrate_complete, reason = self._all_instances_migrated()
|
|
if migrate_complete:
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
# Ensure none of the instances have moved since the strategy step was
|
|
# created. The instance_director.migrate_instances will migrate ALL
|
|
# instances on each host containing one of the self._instance_uuids. We
|
|
# want to ensure we are only migrating instances from the host(s) they
|
|
# were originally located on..
|
|
for instance in self._instances:
|
|
if instance.host_name != self._instance_host_names[instance.uuid]:
|
|
reason = ("instance %s has moved from %s to %s after strategy "
|
|
"created" %
|
|
(instance.name, self._instance_host_names[instance.uuid],
|
|
instance.host_name))
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, reason
|
|
|
|
instance_director = directors.get_instance_director()
|
|
operation = instance_director.migrate_instances(self._instance_uuids)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Instance events
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event in [STRATEGY_EVENT.INSTANCE_STATE_CHANGED,
|
|
STRATEGY_EVENT.INSTANCE_AUDIT]:
|
|
migrate_complete, reason = self._all_instances_migrated()
|
|
|
|
if not migrate_complete and reason:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, reason)
|
|
return True
|
|
|
|
if migrate_complete:
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
elif STRATEGY_EVENT.MIGRATE_INSTANCES_FAILED == event:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, event_data)
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the migrate instances step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(MigrateInstancesStep, self).from_dict(data)
|
|
self._instance_uuids = data['entity_uuids']
|
|
self._instances = list()
|
|
self._instance_names = list()
|
|
self._instance_host_names = dict()
|
|
|
|
instance_table = tables.tables_get_instance_table()
|
|
for instance_uuid in self._instance_uuids:
|
|
instance = instance_table.get(instance_uuid, None)
|
|
if instance is not None:
|
|
self._instances.append(instance)
|
|
self._instance_names.append(instance.name)
|
|
# Retrieve the host this instance was on when the step was
|
|
# created.
|
|
self._instance_host_names[instance.uuid] = \
|
|
data['instance_host_names'][instance.uuid]
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the migrate instances step as a dictionary
|
|
"""
|
|
data = super(MigrateInstancesStep, self).as_dict()
|
|
data['entity_type'] = 'instances'
|
|
data['entity_names'] = self._instance_names
|
|
data['entity_uuids'] = self._instance_uuids
|
|
data['instance_host_names'] = self._instance_host_names
|
|
return data
|
|
|
|
|
|
class StartInstancesStep(strategy.StrategyStep):
|
|
"""
|
|
Start Instances - Strategy Step
|
|
"""
|
|
def __init__(self, instances):
|
|
super(StartInstancesStep, self).__init__(
|
|
STRATEGY_STEP_NAME.START_INSTANCES, timeout_in_secs=900)
|
|
self._instances = instances
|
|
self._instance_names = list()
|
|
self._instance_uuids = list()
|
|
for instance in instances:
|
|
self._instance_names.append(instance.name)
|
|
self._instance_uuids.append(instance.uuid)
|
|
|
|
def _total_instances_unlocked_enabled(self):
|
|
"""
|
|
Returns the number of instances that are unlocked and enabled
|
|
"""
|
|
total_instances_enabled = 0
|
|
instance_table = tables.tables_get_instance_table()
|
|
for instance_uuid in self._instance_uuids:
|
|
instance = instance_table.get(instance_uuid, None)
|
|
if instance is None:
|
|
return -1
|
|
|
|
if instance.is_enabled():
|
|
total_instances_enabled += 1
|
|
|
|
return total_instances_enabled
|
|
|
|
def apply(self):
|
|
"""
|
|
Start all instances
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for instances %s." % (self._name,
|
|
self._instance_names))
|
|
if len(self._instance_uuids) == self._total_instances_unlocked_enabled():
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
instance_director = directors.get_instance_director()
|
|
operation = instance_director.start_instances(self._instance_uuids)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Instance events
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event in [STRATEGY_EVENT.INSTANCE_STATE_CHANGED,
|
|
STRATEGY_EVENT.INSTANCE_AUDIT]:
|
|
total_instances_enabled = self._total_instances_unlocked_enabled()
|
|
|
|
if -1 == total_instances_enabled:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "instance no longer exists")
|
|
return True
|
|
|
|
if total_instances_enabled == len(self._instance_uuids):
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the start instances step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(StartInstancesStep, self).from_dict(data)
|
|
self._instance_uuids = data['entity_uuids']
|
|
self._instances = list()
|
|
self._instance_names = list()
|
|
|
|
instance_table = tables.tables_get_instance_table()
|
|
for instance_uuid in self._instance_uuids:
|
|
instance = instance_table.get(instance_uuid, None)
|
|
if instance is not None:
|
|
self._instances.append(instance)
|
|
self._instance_names.append(instance.name)
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the start instances step as a dictionary
|
|
"""
|
|
data = super(StartInstancesStep, self).as_dict()
|
|
data['entity_type'] = 'instances'
|
|
data['entity_names'] = self._instance_names
|
|
data['entity_uuids'] = self._instance_uuids
|
|
return data
|
|
|
|
|
|
class StopInstancesStep(strategy.StrategyStep):
|
|
"""
|
|
Stop Instances - Strategy Step
|
|
"""
|
|
def __init__(self, instances):
|
|
super(StopInstancesStep, self).__init__(
|
|
STRATEGY_STEP_NAME.STOP_INSTANCES, timeout_in_secs=900)
|
|
self._instances = instances
|
|
self._instance_names = list()
|
|
self._instance_uuids = list()
|
|
self._instance_host_names = dict()
|
|
for instance in instances:
|
|
self._instance_names.append(instance.name)
|
|
self._instance_uuids.append(instance.uuid)
|
|
self._instance_host_names[instance.uuid] = instance.host_name
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
return [StartInstancesStep(self._instances)]
|
|
|
|
def _total_instances_locked_disabled(self):
|
|
"""
|
|
Returns the number of instances that are locked and disabled
|
|
"""
|
|
total_instances_locked = 0
|
|
instance_table = tables.tables_get_instance_table()
|
|
for instance_uuid in self._instance_uuids:
|
|
instance = instance_table.get(instance_uuid, None)
|
|
if instance is None:
|
|
return -1
|
|
|
|
if instance.is_locked() and instance.is_disabled():
|
|
total_instances_locked += 1
|
|
|
|
return total_instances_locked
|
|
|
|
def apply(self):
|
|
"""
|
|
Stop all instances
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for instances %s." % (self._name,
|
|
self._instance_names))
|
|
if len(self._instance_uuids) == self._total_instances_locked_disabled():
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
# Ensure none of the instances have moved since the strategy step was
|
|
# created. There is no point stopping instances that are now on the
|
|
# wrong host.
|
|
for instance in self._instances:
|
|
if instance.host_name != self._instance_host_names[instance.uuid]:
|
|
reason = ("instance %s has moved from %s to %s after strategy "
|
|
"created" %
|
|
(instance.name, self._instance_host_names[instance.uuid],
|
|
instance.host_name))
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, reason
|
|
|
|
instance_director = directors.get_instance_director()
|
|
operation = instance_director.stop_instances(self._instance_uuids)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Instance events
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event in [STRATEGY_EVENT.INSTANCE_STATE_CHANGED,
|
|
STRATEGY_EVENT.INSTANCE_AUDIT]:
|
|
total_instances_locked = self._total_instances_locked_disabled()
|
|
|
|
if -1 == total_instances_locked:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "instance no longer exists")
|
|
return True
|
|
|
|
if total_instances_locked == len(self._instance_uuids):
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the stop instances step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(StopInstancesStep, self).from_dict(data)
|
|
self._instance_uuids = data['entity_uuids']
|
|
self._instances = list()
|
|
self._instance_names = list()
|
|
self._instance_host_names = dict()
|
|
|
|
instance_table = tables.tables_get_instance_table()
|
|
for instance_uuid in self._instance_uuids:
|
|
instance = instance_table.get(instance_uuid, None)
|
|
if instance is not None:
|
|
self._instances.append(instance)
|
|
self._instance_names.append(instance.name)
|
|
|
|
# Retrieve the host this instance was on when the step was
|
|
# created.
|
|
self._instance_host_names[instance.uuid] = \
|
|
data['instance_host_names'][instance.uuid]
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the stop instances step as a dictionary
|
|
"""
|
|
data = super(StopInstancesStep, self).as_dict()
|
|
data['entity_type'] = 'instances'
|
|
data['entity_names'] = self._instance_names
|
|
data['entity_uuids'] = self._instance_uuids
|
|
data['instance_host_names'] = self._instance_host_names
|
|
return data
|
|
|
|
|
|
class SystemStabilizeStep(strategy.StrategyStep):
|
|
"""
|
|
System Stabilize - Strategy Step
|
|
"""
|
|
def __init__(self, timeout_in_secs=60):
|
|
super(SystemStabilizeStep, self).__init__(
|
|
STRATEGY_STEP_NAME.SYSTEM_STABILIZE, timeout_in_secs=timeout_in_secs)
|
|
|
|
def timeout(self):
|
|
"""
|
|
Timeout is expected, so override to pass
|
|
"""
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ''
|
|
|
|
def apply(self):
|
|
"""
|
|
Wait for a period of time
|
|
"""
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host and Instance events
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if STRATEGY_EVENT.HOST_STATE_CHANGED == event:
|
|
host = event_data
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host %s changed state unexpectedly"
|
|
% host.name)
|
|
return True
|
|
|
|
elif STRATEGY_EVENT.INSTANCE_STATE_CHANGED == event:
|
|
instance = event_data
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "instance %s changed state "
|
|
"unexpectedly" % instance.name)
|
|
return True
|
|
|
|
return False
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the system stabilize step as a dictionary
|
|
"""
|
|
data = super(SystemStabilizeStep, self).as_dict()
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = list()
|
|
data['entity_uuids'] = list()
|
|
return data
|
|
|
|
|
|
class QueryAlarmsStep(strategy.StrategyStep):
|
|
"""
|
|
Query Alarms - Strategy Step
|
|
"""
|
|
def __init__(self, fail_on_alarms=False, ignore_alarms=None):
|
|
super(QueryAlarmsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.QUERY_ALARMS, timeout_in_secs=60)
|
|
if ignore_alarms is None:
|
|
ignore_alarms = []
|
|
self._fail_on_alarms = fail_on_alarms
|
|
self._ignore_alarms = ignore_alarms
|
|
|
|
@coroutine
|
|
def _query_alarms_callback(self, fm_service):
|
|
"""
|
|
Query Alarms Callback
|
|
"""
|
|
response = (yield)
|
|
|
|
DLOG.debug("Query-Alarms callback response=%s." % response)
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
nfvi_alarms = self.strategy.nfvi_alarms
|
|
for nfvi_alarm in response['result-data']:
|
|
if (self.strategy._alarm_restrictions ==
|
|
strategy.STRATEGY_ALARM_RESTRICTION_TYPES.RELAXED and
|
|
nfvi_alarm.mgmt_affecting == 'False'):
|
|
DLOG.warn("Ignoring non-management affecting alarm "
|
|
"%s - uuid %s due to relaxed alarm "
|
|
"strictness" % (nfvi_alarm.alarm_id,
|
|
nfvi_alarm.alarm_uuid))
|
|
elif nfvi_alarm.alarm_id not in self._ignore_alarms:
|
|
DLOG.warn("Alarm: %s" % nfvi_alarm.alarm_id)
|
|
nfvi_alarms.append(nfvi_alarm)
|
|
else:
|
|
DLOG.warn("Ignoring alarm %s - uuid %s" %
|
|
(nfvi_alarm.alarm_id, nfvi_alarm.alarm_uuid))
|
|
self.strategy.nfvi_alarms = nfvi_alarms
|
|
|
|
if self._fail_on_alarms and self.strategy.nfvi_alarms:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
alarm_ids = [str(alarm.get('alarm_id')) for alarm in self.strategy.nfvi_alarms]
|
|
reason = "alarms %s from %s are present" % (alarm_ids, fm_service)
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
reason = ""
|
|
|
|
self.stage.step_complete(result, reason)
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
Query Alarms
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
self.strategy.nfvi_alarms = list()
|
|
nfvi.nfvi_get_alarms(self._query_alarms_callback("platform"))
|
|
if not nfvi.nfvi_fault_mgmt_plugin_disabled():
|
|
nfvi.nfvi_get_openstack_alarms(self._query_alarms_callback("openstack"))
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the query alarms step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(QueryAlarmsStep, self).from_dict(data)
|
|
self._fail_on_alarms = data['fail_on_alarms']
|
|
self._ignore_alarms = data['ignore_alarms']
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the query alarms step as a dictionary
|
|
"""
|
|
data = super(QueryAlarmsStep, self).as_dict()
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = list()
|
|
data['entity_uuids'] = list()
|
|
data['fail_on_alarms'] = self._fail_on_alarms
|
|
data['ignore_alarms'] = self._ignore_alarms
|
|
return data
|
|
|
|
|
|
class WaitDataSyncStep(strategy.StrategyStep):
|
|
"""
|
|
Alarm Wait - Strategy Step
|
|
"""
|
|
def __init__(self, timeout_in_secs=300, ignore_alarms=None):
|
|
super(WaitDataSyncStep, self).__init__(
|
|
STRATEGY_STEP_NAME.WAIT_DATA_SYNC, timeout_in_secs=timeout_in_secs)
|
|
if ignore_alarms is None:
|
|
ignore_alarms = []
|
|
self._ignore_alarms = ignore_alarms
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
|
|
@coroutine
|
|
def _query_alarms_callback(self):
|
|
"""
|
|
Query Alarms Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Query-Alarms callback response=%s." % response)
|
|
|
|
self._query_inprogress = False
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
nfvi_alarms = list()
|
|
for nfvi_alarm in response['result-data']:
|
|
if (self.strategy._alarm_restrictions ==
|
|
strategy.STRATEGY_ALARM_RESTRICTION_TYPES.RELAXED and
|
|
nfvi_alarm.mgmt_affecting == 'False'):
|
|
DLOG.warn("Ignoring non-management affecting alarm "
|
|
"%s - uuid %s due to relaxed alarm "
|
|
"strictness" % (nfvi_alarm.alarm_id,
|
|
nfvi_alarm.alarm_uuid))
|
|
elif nfvi_alarm.alarm_id not in self._ignore_alarms:
|
|
nfvi_alarms.append(nfvi_alarm)
|
|
else:
|
|
DLOG.debug("Ignoring alarm %s - uuid %s" %
|
|
(nfvi_alarm.alarm_id, nfvi_alarm.alarm_uuid))
|
|
self.strategy.nfvi_alarms = nfvi_alarms
|
|
|
|
if self.strategy.nfvi_alarms:
|
|
# Keep waiting for alarms to clear
|
|
pass
|
|
else:
|
|
# Alarms have all cleared
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
# Unable to retrieve alarms
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
Alarm Wait
|
|
"""
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
# Wait at least 120 seconds before checking alarms for first time
|
|
if 120 <= secs_expired and not self._query_inprogress:
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_alarms(self._query_alarms_callback())
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the alarm wait step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(WaitDataSyncStep, self).from_dict(data)
|
|
self._ignore_alarms = data['ignore_alarms']
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the alarm wait step as a dictionary
|
|
"""
|
|
data = super(WaitDataSyncStep, self).as_dict()
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = list()
|
|
data['entity_uuids'] = list()
|
|
data['ignore_alarms'] = self._ignore_alarms
|
|
return data
|
|
|
|
|
|
class WaitAlarmsClearStep(strategy.StrategyStep):
|
|
"""
|
|
Alarm Wait - Strategy Step
|
|
"""
|
|
def __init__(self, timeout_in_secs=300, first_query_delay_in_secs=60, ignore_alarms=None):
|
|
super(WaitAlarmsClearStep, self).__init__(
|
|
STRATEGY_STEP_NAME.WAIT_ALARMS_CLEAR, timeout_in_secs=timeout_in_secs)
|
|
self._first_query_delay_in_secs = first_query_delay_in_secs
|
|
if ignore_alarms is None:
|
|
ignore_alarms = []
|
|
self._ignore_alarms = ignore_alarms
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
|
|
@coroutine
|
|
def _query_alarms_callback(self):
|
|
"""
|
|
Query Alarms Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Query-Alarms callback response=%s." % response)
|
|
|
|
self._query_inprogress = False
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
nfvi_alarms = list()
|
|
for nfvi_alarm in response['result-data']:
|
|
if (self.strategy._alarm_restrictions ==
|
|
strategy.STRATEGY_ALARM_RESTRICTION_TYPES.RELAXED and
|
|
nfvi_alarm.mgmt_affecting == 'False'):
|
|
DLOG.warn("Ignoring non-management affecting alarm "
|
|
"%s - uuid %s due to relaxed alarm "
|
|
"strictness" % (nfvi_alarm.alarm_id,
|
|
nfvi_alarm.alarm_uuid))
|
|
elif nfvi_alarm.alarm_id not in self._ignore_alarms:
|
|
nfvi_alarms.append(nfvi_alarm)
|
|
else:
|
|
DLOG.debug("Ignoring alarm %s - uuid %s" %
|
|
(nfvi_alarm.alarm_id, nfvi_alarm.alarm_uuid))
|
|
self.strategy.nfvi_alarms = nfvi_alarms
|
|
|
|
if self.strategy.nfvi_alarms:
|
|
# Keep waiting for alarms to clear
|
|
pass
|
|
else:
|
|
# Alarms have all cleared
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
# Unable to retrieve alarms
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
Alarm Wait
|
|
"""
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
# Wait before checking alarms for first time
|
|
if self._first_query_delay_in_secs <= secs_expired and not self._query_inprogress:
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_alarms(self._query_alarms_callback())
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the alarm wait step object initialized using the given
|
|
dictionary
|
|
"""
|
|
super(WaitAlarmsClearStep, self).from_dict(data)
|
|
self._first_query_delay_in_secs = data['first_query_delay_in_secs']
|
|
self._ignore_alarms = data['ignore_alarms']
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the alarm wait step as a dictionary
|
|
"""
|
|
data = super(WaitAlarmsClearStep, self).as_dict()
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = list()
|
|
data['entity_uuids'] = list()
|
|
data['first_query_delay_in_secs'] = self._first_query_delay_in_secs
|
|
data['ignore_alarms'] = self._ignore_alarms
|
|
return data
|
|
|
|
|
|
class QuerySwPatchesStep(strategy.StrategyStep):
|
|
"""
|
|
Query Software Patches - Strategy Step
|
|
"""
|
|
def __init__(self):
|
|
super(QuerySwPatchesStep, self).__init__(
|
|
STRATEGY_STEP_NAME.QUERY_SW_PATCHES, timeout_in_secs=60)
|
|
|
|
@coroutine
|
|
def _query_sw_patches_callback(self):
|
|
"""
|
|
Query Software Patches Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Query-Sw-Updates callback response=%s." % response)
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_sw_patches = response['result-data']
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
Query Software Patches
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_sw_mgmt_query_updates(self._query_sw_patches_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the query software update step as a dictionary
|
|
"""
|
|
data = super(QuerySwPatchesStep, self).as_dict()
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = list()
|
|
data['entity_uuids'] = list()
|
|
return data
|
|
|
|
|
|
class QuerySwPatchHostsStep(strategy.StrategyStep):
|
|
"""
|
|
Query Software Patch Hosts - Strategy Step
|
|
"""
|
|
def __init__(self):
|
|
super(QuerySwPatchHostsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.QUERY_SW_PATCH_HOSTS, timeout_in_secs=60)
|
|
|
|
@coroutine
|
|
def _query_hosts_callback(self):
|
|
"""
|
|
Query Software Patch Hosts Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Query-Hosts callback response=%s." % response)
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_sw_patch_hosts = response['result-data']
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
Query Software Patch Hosts
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_sw_mgmt_query_hosts(self._query_hosts_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the query software patches hosts step as a dictionary
|
|
"""
|
|
data = super(QuerySwPatchHostsStep, self).as_dict()
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = list()
|
|
data['entity_uuids'] = list()
|
|
return data
|
|
|
|
|
|
class QuerySystemConfigUpdateHostsStep(AbstractStrategyStep):
|
|
"""
|
|
Query System Config Update Hosts - Strategy Step
|
|
"""
|
|
def __init__(self):
|
|
super(QuerySystemConfigUpdateHostsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.QUERY_SYSTEM_CONFIG_UPDATE_HOSTS,
|
|
timeout_in_secs=60)
|
|
|
|
@coroutine
|
|
def _query_hosts_callback(self):
|
|
"""
|
|
Query System Config Update Hosts Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Query-Hosts callback response=%s." % response)
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_system_config_update_hosts = \
|
|
response['result-data']
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
Query System Config Update Hosts
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_list_deployment_hosts(self._query_hosts_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class QueryFwUpdateHostStep(strategy.StrategyStep):
|
|
"""
|
|
Query Host
|
|
"""
|
|
|
|
# This step queries system inventory for the host in self._host_names
|
|
# If the host's 'device_image_update' field shows 'pending' then its
|
|
# hostname is added to the strategy's fw_update_hosts list.
|
|
|
|
def __init__(self, host):
|
|
super(QueryFwUpdateHostStep, self).__init__(
|
|
STRATEGY_STEP_NAME.QUERY_FW_UPDATE_HOST, timeout_in_secs=60)
|
|
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
|
|
@coroutine
|
|
def _get_host_callback(self):
|
|
"""
|
|
Query Host callback
|
|
"""
|
|
|
|
response = (yield)
|
|
|
|
DLOG.verbose("Get-Host %s callback response=%s." %
|
|
(self._host_names[0], response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
hostname = response['result-data'].get('name')
|
|
if hostname:
|
|
device_image_update = response['result-data'].get('device_image_update')
|
|
if device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_PENDING:
|
|
self.strategy.fw_update_hosts.append(hostname)
|
|
DLOG.info("%s requires firmware update" % hostname)
|
|
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_IN_PROGRESS:
|
|
DLOG.info("%s firmware update in-progress" % hostname)
|
|
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_IN_PROGRESS_ABORTED:
|
|
DLOG.info("%s firmware update in-progress-aborted" % hostname)
|
|
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_COMPLETED:
|
|
DLOG.info("%s firmware update complete" % hostname)
|
|
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_FAILED:
|
|
DLOG.info("%s firmware update failed" % hostname)
|
|
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_NULL:
|
|
DLOG.info("%s no firmware update required" % hostname)
|
|
else:
|
|
DLOG.info("%s unknown device_image_update state; %s" %
|
|
(hostname, device_image_update))
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "firmware update query failed")
|
|
|
|
def apply(self):
|
|
"""
|
|
Query Host Apply
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("%s %s step apply" % (self._host_names[0], self._name))
|
|
|
|
# This step is only ever called with one host name.
|
|
nfvi.nfvi_get_host(self._host_uuids[0],
|
|
self._host_names[0],
|
|
self._get_host_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Load the firmware update host device list step
|
|
"""
|
|
super(QueryFwUpdateHostStep, self).from_dict(data)
|
|
self._host_names = data['entity_names']
|
|
self._host_uuids = data['entity_uuids']
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the object as a dictionary for the strategy
|
|
"""
|
|
data = super(QueryFwUpdateHostStep, self).as_dict()
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
return data
|
|
|
|
|
|
class FwUpdateHostsStep(strategy.StrategyStep):
|
|
"""
|
|
Firmware Update Hosts - Strategy Step
|
|
"""
|
|
# This step starts the firmware update process for the passed in hosts
|
|
def __init__(self, hosts):
|
|
super(FwUpdateHostsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.FW_UPDATE_HOSTS, timeout_in_secs=3600)
|
|
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
self._monitoring_fw_update = False
|
|
self._wait_time = 0
|
|
self._host_completed = dict()
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
self._host_completed[host.name] = (False, False, '')
|
|
|
|
@coroutine
|
|
def _get_host_callback(self):
|
|
"""
|
|
Query Host callback used for monitoring update process
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Get-Host callback response=%s." % response)
|
|
try:
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
hostname = response['result-data'].get('name')
|
|
if hostname:
|
|
device_image_update = response['result-data'].get('device_image_update')
|
|
if device_image_update is None:
|
|
DLOG.verbose("%s no firmware update required" % hostname)
|
|
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_PENDING:
|
|
if self._host_completed[hostname][0] is False:
|
|
DLOG.warn("%s firmware update status went pending during update" % hostname)
|
|
failed_msg = hostname + ' firmware update failed ; needs retry'
|
|
self._host_completed[hostname] = (True, False, failed_msg)
|
|
DLOG.error(failed_msg)
|
|
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_IN_PROGRESS:
|
|
DLOG.info("%s firmware update in-progress" % hostname)
|
|
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_IN_PROGRESS_ABORTED:
|
|
if self._host_completed[hostname][0] is False:
|
|
failed_msg = hostname + ' firmware update aborted while in progress'
|
|
self._host_completed[hostname] = (True, False, failed_msg)
|
|
DLOG.error(failed_msg)
|
|
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_COMPLETED or \
|
|
device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_NULL:
|
|
if self._host_completed[hostname][0] is False:
|
|
self._host_completed[hostname] = (True, True, '')
|
|
DLOG.info("%s firmware update complete" % hostname)
|
|
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_FAILED:
|
|
if self._host_completed[hostname][0] is False:
|
|
failed_msg = hostname + ' firmware update failed'
|
|
self._host_completed[hostname] = (True, False, failed_msg)
|
|
DLOG.error(failed_msg)
|
|
else:
|
|
if self._host_completed[hostname][0] is False:
|
|
failed_msg = hostname + \
|
|
' firmware update failed ;' \
|
|
' unknown state [' + \
|
|
device_image_update + ']'
|
|
self._host_completed[hostname] = (True, False, failed_msg)
|
|
DLOG.error(failed_msg)
|
|
|
|
# Check for firmware upgrade step complete
|
|
self._check_step_complete()
|
|
return
|
|
else:
|
|
DLOG.error("failed to get hostname or data from get host response")
|
|
else:
|
|
DLOG.error("failed to monitor firmware update ; no strategy")
|
|
else:
|
|
DLOG.error("get host request did not complete")
|
|
except Exception:
|
|
DLOG.exception("Caught exception interpreting host info")
|
|
DLOG.error("Response: %s" % response)
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
fail_msg = "failed to get or parse fw update info"
|
|
self.stage.step_complete(result, fail_msg)
|
|
|
|
def _check_step_complete(self):
|
|
"""
|
|
Check for firmware upgrade step complete
|
|
"""
|
|
|
|
failed_hosts = ""
|
|
done = True
|
|
for hostname in self._host_names:
|
|
if self._host_completed[hostname][0] is False:
|
|
done = False
|
|
elif self._host_completed[hostname][1] is False:
|
|
failed_hosts += hostname + ' '
|
|
else:
|
|
DLOG.verbose("%s firmware update is complete" % hostname)
|
|
|
|
if done:
|
|
if len(failed_hosts) == 0:
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
failed_msg = 'Firmware update failed ; %s' % failed_hosts
|
|
self.stage.step_complete(result, failed_msg)
|
|
|
|
def apply(self):
|
|
"""
|
|
Firmware Update Hosts Apply
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
|
|
self._host_names))
|
|
|
|
if len(self._host_names):
|
|
host_director = directors.get_host_director()
|
|
operation = host_director.fw_update_hosts(self._host_names)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
else:
|
|
reason = "no hosts found in firmware update step"
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, reason)
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, reason
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Firmware Image Update events
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_FW_UPDATE_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "fw image update failed")
|
|
return True
|
|
|
|
elif event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if not self._monitoring_fw_update:
|
|
self._monitoring_fw_update = True
|
|
DLOG.info("Start monitoring firmware update progress for %s" %
|
|
self._host_names)
|
|
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
if 60 <= secs_expired:
|
|
# force timer reload on next audit
|
|
self._wait_time = 0
|
|
|
|
for host in self._hosts:
|
|
if self._host_completed[host.name][0] is True:
|
|
DLOG.info("%s firmware update already done ; pass=%s" %
|
|
(host.name,
|
|
self._host_completed[host.name][1]))
|
|
continue
|
|
|
|
nfvi.nfvi_get_host(host.uuid,
|
|
host.name,
|
|
self._get_host_callback())
|
|
return True
|
|
else:
|
|
DLOG.warn("Unexpected event (%s)" % event)
|
|
|
|
return False
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step with applicable host list
|
|
"""
|
|
|
|
# abort all hosts that are not in the completed state
|
|
hosts = list()
|
|
for host in self._hosts:
|
|
if self._host_completed[host.name][0] is False:
|
|
hosts.append(host)
|
|
return [FwUpdateAbortHostsStep(hosts)]
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the firmware update hosts step object
|
|
initialized using the given dictionary
|
|
"""
|
|
super(FwUpdateHostsStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_completed = dict()
|
|
self._wait_time = 0
|
|
self._monitoring_fw_update = False
|
|
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
self._host_completed[host_name] = \
|
|
data['hosts_completed'][host_name]
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the firmware update hosts step as a dictionary
|
|
"""
|
|
data = super(FwUpdateHostsStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
data['hosts_completed'] = self._host_completed
|
|
return data
|
|
|
|
|
|
class FwUpdateAbortHostsStep(strategy.StrategyStep):
|
|
"""
|
|
Firmware Update Abort Hosts Step
|
|
"""
|
|
def __init__(self, hosts):
|
|
super(FwUpdateAbortHostsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.FW_UPDATE_ABORT_HOSTS, timeout_in_secs=600)
|
|
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
|
|
self._wait_time = 0
|
|
|
|
self._host_completed = dict()
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
self._host_completed[host.name] = (False, False, '')
|
|
|
|
def apply(self):
|
|
"""
|
|
Monitor Firmware Update Abort Hosts Apply
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
|
|
self._host_names))
|
|
|
|
host_director = directors.get_host_director()
|
|
operation = host_director.fw_update_abort_hosts(self._host_names)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Firmware Image Update Abort events
|
|
"""
|
|
# from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.HOST_FW_UPDATE_ABORT_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
failed_msg = "device image update abort failed"
|
|
DLOG.info("%s %s" % (host.name, failed_msg))
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, failed_msg)
|
|
return True
|
|
|
|
elif event == STRATEGY_EVENT.HOST_AUDIT:
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Load the firmware update abort hosts step object
|
|
"""
|
|
super(FwUpdateAbortHostsStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_completed = dict()
|
|
self._wait_time = 0
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
self._host_completed[host_name] = \
|
|
data['hosts_completed'][host_name]
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Save the firmware update abort hosts step as a dictionary
|
|
"""
|
|
data = super(FwUpdateAbortHostsStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
data['hosts_completed'] = self._host_completed
|
|
return data
|
|
|
|
|
|
class QueryUpgradeStep(strategy.StrategyStep):
|
|
"""
|
|
Query Upgrade - Strategy Step
|
|
"""
|
|
def __init__(self):
|
|
super(QueryUpgradeStep, self).__init__(
|
|
STRATEGY_STEP_NAME.QUERY_UPGRADE, timeout_in_secs=60)
|
|
|
|
@coroutine
|
|
def _get_upgrade_callback(self):
|
|
"""
|
|
Get Upgrade Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("Query-Upgrade callback response=%s." % response)
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_upgrade = response['result-data']
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "")
|
|
|
|
def apply(self):
|
|
"""
|
|
Query Software Upgrade
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the query upgrade step as a dictionary
|
|
"""
|
|
data = super(QueryUpgradeStep, self).as_dict()
|
|
data['entity_type'] = ''
|
|
data['entity_names'] = list()
|
|
data['entity_uuids'] = list()
|
|
return data
|
|
|
|
|
|
class DisableHostServicesStep(strategy.StrategyStep):
|
|
"""
|
|
Disable Host Services - Strategy Step
|
|
"""
|
|
def __init__(self, hosts, service):
|
|
super(DisableHostServicesStep, self).__init__(
|
|
"%s" % STRATEGY_STEP_NAME.DISABLE_HOST_SERVICES,
|
|
timeout_in_secs=180)
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
self._service = service
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
return [EnableHostServicesStep(self._hosts, self._service)]
|
|
|
|
def _total_hosts_services_disabled(self):
|
|
"""
|
|
Returns the number of hosts with services disabled
|
|
"""
|
|
total_hosts_services_disabled = 0
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is None:
|
|
return -1
|
|
|
|
if (objects.HOST_SERVICE_STATE.DISABLED ==
|
|
host.host_service_state(self._service)):
|
|
total_hosts_services_disabled += 1
|
|
|
|
return total_hosts_services_disabled
|
|
|
|
def apply(self):
|
|
"""
|
|
Disable host services on specified hosts
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for hosts %s service %s." %
|
|
(self._name, self._host_names, self._service))
|
|
host_director = directors.get_host_director()
|
|
operation = host_director.disable_host_services(self._host_names,
|
|
self._service)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event in [STRATEGY_EVENT.HOST_STATE_CHANGED,
|
|
STRATEGY_EVENT.HOST_AUDIT]:
|
|
total_hosts_services_disabled = \
|
|
self._total_hosts_services_disabled()
|
|
|
|
if -1 == total_hosts_services_disabled:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host no longer exists")
|
|
return True
|
|
|
|
if total_hosts_services_disabled == len(self._host_names):
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
elif event == STRATEGY_EVENT.DISABLE_HOST_SERVICES_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result,
|
|
"disable host services failed")
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the object initialized using the given dictionary
|
|
"""
|
|
super(DisableHostServicesStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_names = data['entity_names']
|
|
self._service = data['entity_service']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the object as a dictionary
|
|
"""
|
|
data = super(DisableHostServicesStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
data['entity_service'] = self._service
|
|
return data
|
|
|
|
|
|
class EnableHostServicesStep(strategy.StrategyStep):
|
|
"""
|
|
Enable Host Services - Strategy Step
|
|
"""
|
|
def __init__(self, hosts, service):
|
|
super(EnableHostServicesStep, self).__init__(
|
|
"%s" % STRATEGY_STEP_NAME.ENABLE_HOST_SERVICES,
|
|
timeout_in_secs=180)
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
self._service = service
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
|
|
def _total_hosts_services_enabled(self):
|
|
"""
|
|
Returns the number of hosts with services enabled
|
|
"""
|
|
total_hosts_services_enabled = 0
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is None:
|
|
return -1
|
|
|
|
if (objects.HOST_SERVICE_STATE.ENABLED ==
|
|
host.host_service_state(self._service)):
|
|
total_hosts_services_enabled += 1
|
|
|
|
return total_hosts_services_enabled
|
|
|
|
def apply(self):
|
|
"""
|
|
Enable host services on specified hosts
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply for hosts %s service %s." %
|
|
(self._name, self._host_names, self._service))
|
|
host_director = directors.get_host_director()
|
|
operation = host_director.enable_host_services(self._host_names,
|
|
self._service)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event in [STRATEGY_EVENT.HOST_STATE_CHANGED,
|
|
STRATEGY_EVENT.HOST_AUDIT]:
|
|
total_hosts_services_enabled = \
|
|
self._total_hosts_services_enabled()
|
|
|
|
if -1 == total_hosts_services_enabled:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "host no longer exists")
|
|
return True
|
|
|
|
if total_hosts_services_enabled == len(self._host_names):
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, '')
|
|
return True
|
|
|
|
elif event == STRATEGY_EVENT.ENABLE_HOST_SERVICES_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "enable host services failed")
|
|
return True
|
|
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the object initialized using the given dictionary
|
|
"""
|
|
super(EnableHostServicesStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_names = data['entity_names']
|
|
self._service = data['entity_service']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the object as a dictionary
|
|
"""
|
|
data = super(EnableHostServicesStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
data['entity_service'] = self._service
|
|
return data
|
|
|
|
|
|
class ApplySwPatchesStep(AbstractStrategyStep):
|
|
"""
|
|
Apply Patches using patch API
|
|
"""
|
|
def __init__(self, patches_to_apply):
|
|
super(ApplySwPatchesStep, self).__init__(
|
|
STRATEGY_STEP_NAME.APPLY_PATCHES,
|
|
timeout_in_secs=600)
|
|
self._patches_to_apply = patches_to_apply
|
|
|
|
@coroutine
|
|
def _api_callback(self):
|
|
"""
|
|
Callback for the API method invoked in apply
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_sw_patches = response['result-data']
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""
|
|
Apply patches
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
nfvi.nfvi_sw_mgmt_apply_updates(self._patches_to_apply,
|
|
self._api_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(ApplySwPatchesStep, self).from_dict(data)
|
|
# only the names are serialized
|
|
self._patches_to_apply = data['entity_names']
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the step as a dictionary
|
|
"""
|
|
data = super(ApplySwPatchesStep, self).as_dict()
|
|
data['entity_type'] = 'patches'
|
|
data['entity_names'] = self._patches_to_apply
|
|
# there are no entity_uuids
|
|
return data
|
|
|
|
|
|
##########################################################
|
|
# Kube RootCA Update Steps
|
|
##########################################################
|
|
class QueryKubeRootcaUpdateStep(AbstractStrategyStep):
|
|
"""
|
|
Query Kube RootCA Update
|
|
"""
|
|
def __init__(self):
|
|
super(QueryKubeRootcaUpdateStep, self).__init__(
|
|
STRATEGY_STEP_NAME.QUERY_KUBE_ROOTCA_UPDATE,
|
|
timeout_in_secs=60)
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""
|
|
Get Kube Root CA Update Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_rootca_update = response['result-data']
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""
|
|
Query Kube Root CA Update
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_get_kube_rootca_update(self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class QueryKubeRootcaHostUpdatesStep(AbstractStrategyStep):
|
|
"""
|
|
Query Kube Rootca Host Update list
|
|
"""
|
|
def __init__(self):
|
|
super(QueryKubeRootcaHostUpdatesStep, self).__init__(
|
|
STRATEGY_STEP_NAME.QUERY_KUBE_ROOTCA_HOST_UPDATES,
|
|
timeout_in_secs=60)
|
|
|
|
@coroutine
|
|
def _get_kube_rootca_host_update_list_callback(self):
|
|
"""
|
|
Get Kube RootCA Host Update List Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_rootca_host_update_list = \
|
|
response['result-data']
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""
|
|
Query Kube Host Upgrade List
|
|
"""
|
|
# todo(abailey): The API raises an exception if the kube rootca hosts
|
|
# are queried when there is no active update in progress
|
|
# therefore there are additional checks here
|
|
if self.strategy is None:
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, "No strategy"
|
|
if self.strategy.nfvi_kube_rootca_update is None:
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, "No update"
|
|
|
|
from nfv_vim import nfvi
|
|
nfvi.nfvi_get_kube_rootca_host_update_list(
|
|
self._get_kube_rootca_host_update_list_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class AbstractKubeRootcaUpdateStep(AbstractStrategyStep):
|
|
"""
|
|
Abstract Step class for processing changes to kube root ca update
|
|
"""
|
|
# todo(abailey): The hosts and pod steps have in_progress_state and
|
|
# fail_state but the majority of the transition steps do not so there
|
|
# should be an abstract class defined above this which adds those.
|
|
|
|
def __init__(self,
|
|
step_name,
|
|
success_state,
|
|
in_progress_state,
|
|
fail_state,
|
|
timeout_in_secs=600):
|
|
super(AbstractKubeRootcaUpdateStep, self).__init__(step_name,
|
|
timeout_in_secs)
|
|
# _wait_time and _query_inprogress are NOT persisted
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
# success, in-progress and fail state validators are persisted
|
|
self._success_state = success_state
|
|
self._in_progress_state = in_progress_state
|
|
self._fail_state = fail_state
|
|
|
|
@coroutine
|
|
def _get_kube_rootca_update_callback(self):
|
|
"""
|
|
Get Kube RootCA Update - Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("(%s) callback response=%s." % (self._name, response))
|
|
|
|
self._query_inprogress = False
|
|
if response['completed']:
|
|
if self.strategy is None:
|
|
# there is no longer a strategy. abort.
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, 'strategy no longer exists')
|
|
|
|
result_obj = response['result-data']
|
|
# replace the object in the strategy with the most recent object
|
|
self.strategy.nfvi_kube_rootca_update = result_obj
|
|
|
|
if result_obj is None:
|
|
if self._success_state is None:
|
|
DLOG.debug("(%s) successfully cleared" % self._name)
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
DLOG.debug("(%s) unexpectedly cleared" % self._name)
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, "Result was cleared")
|
|
# break out of the loop if fail or success states match
|
|
elif result_obj.state == self._success_state:
|
|
DLOG.debug("(%s) successfully reached (%s)."
|
|
% (self._name, self._success_state))
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
elif (self._fail_state is not None
|
|
and result_obj.state == self._fail_state):
|
|
DLOG.warn("(%s) encountered failure state(%s)."
|
|
% (self._name, self._fail_state))
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(
|
|
result,
|
|
'(%s) failed:(%s)' % (self._name, self._fail_state)
|
|
)
|
|
else:
|
|
# Keep waiting for object to reach success or fail state
|
|
# timeout will occur if it is never reached.
|
|
DLOG.debug("(%s) in state (%s) waiting for (%s) or (%s)."
|
|
% (self._name,
|
|
result_obj.state,
|
|
self._success_state,
|
|
self._fail_state))
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""Handle Host events"""
|
|
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
if event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
# Wait at least 60 seconds before checking update for first time
|
|
if 60 <= secs_expired and not self._query_inprogress:
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_kube_rootca_update(
|
|
self._get_kube_rootca_update_callback())
|
|
return True
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(AbstractKubeRootcaUpdateStep, self).from_dict(data)
|
|
# these two attributes are not persisted
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
# validation states are persisted
|
|
self._success_state = data['success_state']
|
|
self._in_progress_state = data['in_progress_state']
|
|
self._fail_state = data['fail_state']
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the step as a dictionary
|
|
"""
|
|
data = super(AbstractKubeRootcaUpdateStep, self).as_dict()
|
|
data['success_state'] = self._success_state
|
|
data['in_progress_state'] = self._in_progress_state
|
|
data['fail_state'] = self._fail_state
|
|
return data
|
|
|
|
|
|
class AbstractKubeRootcaUpdateHostStep(AbstractKubeRootcaUpdateStep):
|
|
def __init__(self,
|
|
hosts,
|
|
step_name,
|
|
success_state,
|
|
in_progress_state,
|
|
fail_state,
|
|
update_type,
|
|
timeout_in_secs=600):
|
|
super(AbstractKubeRootcaUpdateHostStep, self).__init__(
|
|
step_name,
|
|
success_state,
|
|
in_progress_state,
|
|
fail_state,
|
|
timeout_in_secs=timeout_in_secs)
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
self._update_type = update_type
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(AbstractKubeRootcaUpdateHostStep, self).from_dict(data)
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
self._update_type = data['update_type']
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the step as a dictionary
|
|
"""
|
|
data = super(AbstractKubeRootcaUpdateHostStep, self).as_dict()
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
data['update_type'] = self._update_type
|
|
return data
|
|
|
|
@coroutine
|
|
def _get_kube_rootca_host_update_list_callback(self):
|
|
"""Get Kube RootCa Update Host List Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("(%s) callback response=%s." % (self._name, response))
|
|
|
|
self._query_inprogress = False
|
|
if response['completed']:
|
|
self.strategy.nfvi_kube_rootca_host_update_list = \
|
|
response['result-data']
|
|
|
|
host_count = 0
|
|
match_count = 0
|
|
fail_count = 0
|
|
for host_name in self._host_names:
|
|
for k_host in self.strategy.nfvi_kube_rootca_host_update_list:
|
|
if k_host.hostname == host_name:
|
|
if k_host.state == self._success_state:
|
|
match_count += 1
|
|
elif k_host.state == self._fail_state:
|
|
# we should not have gotten here
|
|
fail_count += 1
|
|
# todo(abailey): We can add an in-progress check here
|
|
# and treat as failed if the time is too long
|
|
host_count += 1
|
|
# break out of inner loop, since uuids match
|
|
break
|
|
if host_count == len(self._host_uuids):
|
|
# this is a pointless break
|
|
break
|
|
if match_count == len(self._host_uuids):
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
elif fail_count != 0:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
else:
|
|
# keep waiting for state to change
|
|
pass
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events - check for a host failure event
|
|
Override to bypass checking for kube rootca update state.
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.KUBE_ROOTCA_UPDATE_HOST_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(
|
|
result,
|
|
"%s for host(%s) failed"
|
|
% (self._name, host.name))
|
|
return True
|
|
elif event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
# Wait at least 60 seconds before checking update for first time
|
|
if 60 <= secs_expired and not self._query_inprogress:
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_kube_rootca_host_update_list(
|
|
self._get_kube_rootca_host_update_list_callback())
|
|
return True
|
|
return False
|
|
|
|
def apply(self):
|
|
"""Kube RootCA Update Hosts"""
|
|
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply to hostnames (%s)."
|
|
% (self._name, self._host_names))
|
|
host_director = directors.get_host_director()
|
|
operation = host_director.kube_rootca_update_hosts_by_type(
|
|
self._host_names,
|
|
self._update_type,
|
|
self._in_progress_state,
|
|
self._success_state,
|
|
self._fail_state)
|
|
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
|
|
class KubeRootcaUpdateHostTrustBothcasStep(AbstractKubeRootcaUpdateHostStep):
|
|
"""Kube RootCA Update - Host - trustBothCAs"""
|
|
|
|
def __init__(self, hosts):
|
|
from nfv_vim import nfvi
|
|
super(KubeRootcaUpdateHostTrustBothcasStep, self).__init__(
|
|
hosts,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_HOST_TRUSTBOTHCAS,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATED_HOST_TRUSTBOTHCAS,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS_FAILED,
|
|
KUBE_CERT_UPDATE_TRUSTBOTHCAS)
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
return [KubeRootcaUpdateAbortStep()]
|
|
|
|
|
|
class KubeRootcaUpdateHostUpdateCertsStep(AbstractKubeRootcaUpdateHostStep):
|
|
"""Kube RootCA Update - Host - updateCerts"""
|
|
|
|
def __init__(self, hosts):
|
|
from nfv_vim import nfvi
|
|
super(KubeRootcaUpdateHostUpdateCertsStep, self).__init__(
|
|
hosts,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_HOST_UPDATECERTS,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS_FAILED,
|
|
KUBE_CERT_UPDATE_UPDATECERTS)
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
return [KubeRootcaUpdateAbortStep()]
|
|
|
|
|
|
class KubeRootcaUpdateHostTrustNewcaStep(AbstractKubeRootcaUpdateHostStep):
|
|
"""Kube RootCA Update - Host - trustNewCA"""
|
|
|
|
def __init__(self, hosts):
|
|
from nfv_vim import nfvi
|
|
super(KubeRootcaUpdateHostTrustNewcaStep, self).__init__(
|
|
hosts,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_HOST_TRUSTNEWCA,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATED_HOST_TRUSTNEWCA,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA_FAILED,
|
|
KUBE_CERT_UPDATE_TRUSTNEWCA)
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
return [KubeRootcaUpdateAbortStep()]
|
|
|
|
|
|
class AbstractKubeRootcaUpdatePodsStep(AbstractKubeRootcaUpdateStep):
|
|
"""
|
|
Abstract Step class for processing changes to kube rootca update for pods
|
|
"""
|
|
|
|
def __init__(self,
|
|
step_name,
|
|
success_state,
|
|
in_progress_state,
|
|
fail_state,
|
|
phase,
|
|
timeout_in_secs=3600):
|
|
super(AbstractKubeRootcaUpdatePodsStep, self).__init__(
|
|
step_name,
|
|
success_state,
|
|
in_progress_state,
|
|
fail_state,
|
|
timeout_in_secs=timeout_in_secs)
|
|
self._phase = phase
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(AbstractKubeRootcaUpdatePodsStep, self).from_dict(data)
|
|
self._phase = data['phase']
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the step as a dictionary
|
|
"""
|
|
data = super(AbstractKubeRootcaUpdatePodsStep, self).as_dict()
|
|
data['phase'] = self._phase
|
|
return data
|
|
|
|
@coroutine
|
|
def _pods_update_response_callback(self):
|
|
"""Kube RootCA Update - Pods - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_rootca_update = response['result-data']
|
|
# We do not set 'success' here, let the handle_event do this
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube RootCA Update - Pods"""
|
|
from nfv_vim import nfvi
|
|
nfvi.nfvi_kube_rootca_update_pods(self._phase,
|
|
self._pods_update_response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class KubeRootcaUpdatePodsTrustBothcasStep(AbstractKubeRootcaUpdatePodsStep):
|
|
"""Kube RootCA Update - Pods - trustBothCAs - Strategy Step"""
|
|
|
|
def __init__(self):
|
|
from nfv_vim import nfvi
|
|
super(KubeRootcaUpdatePodsTrustBothcasStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_PODS_TRUSTBOTHCAS,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATED_PODS_TRUSTBOTHCAS,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS_FAILED,
|
|
KUBE_CERT_UPDATE_TRUSTBOTHCAS) # phase
|
|
|
|
|
|
class KubeRootcaUpdatePodsTrustNewcaStep(AbstractKubeRootcaUpdatePodsStep):
|
|
"""Kube RootCA Update - Pods - trustNewCA - Strategy Step"""
|
|
|
|
def __init__(self):
|
|
from nfv_vim import nfvi
|
|
super(KubeRootcaUpdatePodsTrustNewcaStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_PODS_TRUSTNEWCA,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA_FAILED,
|
|
KUBE_CERT_UPDATE_TRUSTNEWCA) # phase
|
|
|
|
|
|
class KubeRootcaUpdateStartStep(AbstractKubeRootcaUpdateStep):
|
|
"""Kube RootCA Update - Start - Strategy Step"""
|
|
|
|
def __init__(self):
|
|
from nfv_vim import nfvi
|
|
super(KubeRootcaUpdateStartStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_START,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATE_STARTED,
|
|
None, # sysinv API does not have in-progress state for this action
|
|
None) # there is no failure state if 'start' fails
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Kube RootCA Update - Start - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_rootca_update = response['result-data']
|
|
# We do not set 'success' here, let the handle_event do this
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube RootCA Update - Start"""
|
|
|
|
from nfv_vim import nfvi
|
|
|
|
force = True # Ignore all non-management affecting alarms
|
|
alarm_ignore_list = ["900.501", ] # ignore the auto apply alarm
|
|
nfvi.nfvi_kube_rootca_update_start(force,
|
|
alarm_ignore_list,
|
|
self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class KubeRootcaUpdateAbortStep(AbstractKubeRootcaUpdateStep):
|
|
"""Kube RootCA Update - Abort - Strategy Step"""
|
|
|
|
def __init__(self):
|
|
from nfv_vim import nfvi
|
|
super(KubeRootcaUpdateAbortStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_ABORT,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATE_ABORTED,
|
|
None, # sysinv API does not have in-progress state for this action
|
|
None) # there is no failure state if 'abort' fails
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Kube RootCA Update - Abort - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_rootca_update = response['result-data']
|
|
|
|
# Calling abort on an aborted update returns a failure so we check
|
|
# the rootca update object on success AND failure.
|
|
if self.strategy is None:
|
|
# return success if there is no more strategy
|
|
self.stage.step_complete(strategy.STRATEGY_STEP_RESULT.SUCCESS,
|
|
"no strategy")
|
|
elif self.strategy.nfvi_kube_rootca_update is None:
|
|
# return success if there is no more update
|
|
self.stage.step_complete(strategy.STRATEGY_STEP_RESULT.SUCCESS,
|
|
"no update")
|
|
elif self.strategy.nfvi_kube_rootca_update.state == self._success_state:
|
|
self.stage.step_complete(strategy.STRATEGY_STEP_RESULT.SUCCESS,
|
|
"")
|
|
else:
|
|
# If the state does not match, the abort failed.
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result,
|
|
"Unexpected state: %s"
|
|
% self.strategy.nfvi_kube_rootca_update.state)
|
|
|
|
def apply(self):
|
|
"""Kube RootCA Update - Abort"""
|
|
|
|
from nfv_vim import nfvi
|
|
|
|
nfvi.nfvi_kube_rootca_update_abort(self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class KubeRootcaUpdateCompleteStep(AbstractKubeRootcaUpdateStep):
|
|
"""Kube RootCA Update - Complete - Strategy Step"""
|
|
|
|
def __init__(self):
|
|
from nfv_vim import nfvi
|
|
super(KubeRootcaUpdateCompleteStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_COMPLETE,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATE_COMPLETED,
|
|
None, # sysinv API does not have in-progress state for this action
|
|
None) # there is no failure state if 'complete' fails
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""Override parent class and do not handle any events"""
|
|
return False
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Kube RootCA Update - Complete - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_rootca_update = response['result-data']
|
|
if self.strategy is None:
|
|
# return success if there is no more strategy
|
|
self.stage.step_complete(strategy.STRATEGY_STEP_RESULT.SUCCESS,
|
|
"no strategy")
|
|
elif self.strategy.nfvi_kube_rootca_update is None:
|
|
# return success if there is no more update
|
|
self.stage.step_complete(strategy.STRATEGY_STEP_RESULT.SUCCESS,
|
|
"no update")
|
|
elif self.strategy.nfvi_kube_rootca_update.state == self._success_state:
|
|
self.stage.step_complete(strategy.STRATEGY_STEP_RESULT.SUCCESS,
|
|
"")
|
|
else:
|
|
# We should not get here.
|
|
# If the state does not match, the response should have failed
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result,
|
|
"Unexpected state: %s"
|
|
% self.strategy.nfvi_kube_rootca_update.state)
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube RootCA Update - Complete"""
|
|
|
|
from nfv_vim import nfvi
|
|
|
|
nfvi.nfvi_kube_rootca_update_complete(self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class KubeRootcaUpdateGenerateCertStep(AbstractKubeRootcaUpdateStep):
|
|
"""Kube RootCA Update - Generate Cert - Strategy Step"""
|
|
|
|
def __init__(self, expiry_date, subject):
|
|
from nfv_vim import nfvi
|
|
super(KubeRootcaUpdateGenerateCertStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_GENERATE_CERT,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATE_CERT_GENERATED,
|
|
None, # sysinv API does not have in-progress state for this action
|
|
None, # sysinv API does not have a FAILED state for this action
|
|
timeout_in_secs=300) # set a five minute timeout to detect failure
|
|
self._expiry_date = expiry_date
|
|
self._subject = subject
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Generate Cert is a blocking call that returns an identifier
|
|
Polling the update is how we proceed to next state
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
if response['completed']:
|
|
# We do not set 'success' here, let the handle_event do this
|
|
# The API returns a certificate identifier
|
|
pass
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube RootCA Update - Generate Cert"""
|
|
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_kube_rootca_update_generate_cert(self._expiry_date,
|
|
self._subject,
|
|
self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(KubeRootcaUpdateGenerateCertStep, self).from_dict(data)
|
|
self._expiry_date = data['expiry_date']
|
|
self._subject = data['subject']
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the kube upgrade step as a dictionary
|
|
"""
|
|
data = super(KubeRootcaUpdateGenerateCertStep, self).as_dict()
|
|
data['expiry_date'] = self._expiry_date
|
|
data['subject'] = self._subject
|
|
return data
|
|
|
|
|
|
class KubeRootcaUpdateUploadCertStep(AbstractKubeRootcaUpdateStep):
|
|
"""Kube RootCA Update - Upload Cert - Strategy Step"""
|
|
|
|
def __init__(self, cert_file):
|
|
from nfv_vim import nfvi
|
|
super(KubeRootcaUpdateUploadCertStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_UPLOAD_CERT,
|
|
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATE_CERT_UPLOADED,
|
|
None, # sysinv API does not have in-progress state for this action
|
|
None, # sysinv API does not have a FAILED state for this action
|
|
timeout_in_secs=300)
|
|
self._cert_file = cert_file
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Upload Cert is a blocking call that returns an identifier
|
|
Polling the update is how we proceed to next state
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
if response['completed']:
|
|
# We do not set 'success' here, let the handle_event do this
|
|
# The API returns a certificate identifier
|
|
pass
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube RootCA Update - Upload Cert"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_kube_rootca_update_upload_cert(self._cert_file,
|
|
self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(KubeRootcaUpdateUploadCertStep, self).from_dict(data)
|
|
self._cert_file = data['cert_file']
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the kube upgrade step as a dictionary
|
|
"""
|
|
data = super(KubeRootcaUpdateUploadCertStep, self).as_dict()
|
|
data['cert_file'] = self._cert_file
|
|
return data
|
|
|
|
|
|
##########################################################
|
|
# Kube Upgrade Steps
|
|
##########################################################
|
|
class QueryKubeUpgradeStep(AbstractStrategyStep):
|
|
"""
|
|
Query Kube Upgrade
|
|
"""
|
|
def __init__(self):
|
|
super(QueryKubeUpgradeStep, self).__init__(
|
|
STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE, timeout_in_secs=60)
|
|
|
|
@coroutine
|
|
def _get_kube_upgrade_callback(self):
|
|
"""
|
|
Get Kube Upgrade Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_upgrade = response['result-data']
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""
|
|
Query Kube Upgrade
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_get_kube_upgrade(self._get_kube_upgrade_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class QueryKubeVersionsStep(AbstractStrategyStep):
|
|
"""
|
|
Query Kube Versions
|
|
This step should be used with its matching QueryKubeVersionsMixin
|
|
"""
|
|
def __init__(self):
|
|
super(QueryKubeVersionsStep, self).__init__(
|
|
STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS, timeout_in_secs=60)
|
|
|
|
@coroutine
|
|
def _query_callback(self):
|
|
"""
|
|
Get Kube Versions List Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_versions_list = response['result-data']
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""
|
|
Query Kube Versions List
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_get_kube_version_list(self._query_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class QueryKubeHostUpgradeStep(AbstractStrategyStep):
|
|
"""
|
|
Query Kube Host Upgrade list
|
|
"""
|
|
MAX_RETRIES = 3
|
|
|
|
def __init__(self, retry_count=MAX_RETRIES):
|
|
super(QueryKubeHostUpgradeStep, self).__init__(
|
|
STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE, timeout_in_secs=200)
|
|
self._retry_count = retry_count
|
|
|
|
@coroutine
|
|
def _get_kube_host_upgrade_list_callback(self):
|
|
"""
|
|
Get Kube Host Upgrade List Callback
|
|
"""
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_host_upgrade_list = \
|
|
response['result-data']
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""
|
|
Query Kube Host Upgrade List
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
|
|
host_director = directors.get_host_director()
|
|
host_director._nfvi_get_kube_host_upgrade_list()
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Query Kube Host upgrade event
|
|
"""
|
|
from nfv_vim import directors
|
|
|
|
if event == STRATEGY_EVENT.QUERY_KUBE_HOST_UPGRADE_FAILED:
|
|
if event_data is not None and self._retry_count > 0:
|
|
# if kube host upgrade list fails and we have retries,
|
|
# re-trigger the function
|
|
DLOG.info("Step (%s) retry due to failure for (%s)." % (self._name,
|
|
str(event_data["reason"])))
|
|
|
|
self._retry_count = self._retry_count - 1
|
|
host_director = directors.get_host_director()
|
|
host_director._nfvi_get_kube_host_upgrade_list()
|
|
else:
|
|
# if kube host upgrade list fails and we are out of retries, fail
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, event_data['reason'])
|
|
return True
|
|
|
|
elif event == STRATEGY_EVENT.QUERY_KUBE_HOST_UPGRADE_COMPLETED:
|
|
if event_data is not None and self.strategy is not None:
|
|
self.strategy.nfvi_kube_host_upgrade_list = \
|
|
event_data['result-data']
|
|
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
|
|
return False
|
|
|
|
|
|
class AbstractKubeUpgradeStep(AbstractStrategyStep):
|
|
|
|
def __init__(self,
|
|
step_name,
|
|
success_state,
|
|
fail_state,
|
|
timeout_in_secs=600):
|
|
super(AbstractKubeUpgradeStep, self).__init__(step_name,
|
|
timeout_in_secs)
|
|
# These two attributes are not persisted
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
# success and fail state validators are persisted
|
|
self._success_state = success_state
|
|
self._fail_state = fail_state
|
|
|
|
@coroutine
|
|
def _get_kube_upgrade_callback(self):
|
|
"""Get Upgrade Callback"""
|
|
response = (yield)
|
|
DLOG.debug("(%s) callback response=%s." % (self._name, response))
|
|
|
|
self._query_inprogress = False
|
|
if response['completed']:
|
|
if self.strategy is None:
|
|
# there is no longer a strategy. abort.
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, 'strategy no longer exists')
|
|
|
|
kube_upgrade_obj = response['result-data']
|
|
# replace the object in the strategy with the most recent object
|
|
self.strategy.nfvi_kube_upgrade = kube_upgrade_obj
|
|
|
|
# break out of the loop if fail or success states match
|
|
if kube_upgrade_obj and kube_upgrade_obj.state == self._success_state:
|
|
DLOG.debug("(%s) successfully reached (%s)."
|
|
% (self._name, self._success_state))
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
elif (self._fail_state is not None
|
|
and kube_upgrade_obj.state == self._fail_state):
|
|
DLOG.warn("(%s) encountered failure state(%s)."
|
|
% (self._name, self._fail_state))
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(
|
|
result,
|
|
'(%s) failed:(%s)' % (self._name, self._fail_state)
|
|
)
|
|
else:
|
|
# Keep waiting for upgrade to reach success or fail state
|
|
# timeout will occur if it is never reached.
|
|
if kube_upgrade_obj:
|
|
kube_upgrade_obj_state = kube_upgrade_obj.state
|
|
else:
|
|
kube_upgrade_obj_state = None
|
|
DLOG.debug("(%s) in state (%s) waiting for (%s) or (%s)."
|
|
% (self._name,
|
|
kube_upgrade_obj_state,
|
|
self._success_state,
|
|
self._fail_state))
|
|
pass
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""Handle Host events"""
|
|
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
if event == STRATEGY_EVENT.KUBE_UPGRADE_CHANGED:
|
|
# todo(abailey): use event data rather than re-querying
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_kube_upgrade(self._get_kube_upgrade_callback())
|
|
return True
|
|
elif event == STRATEGY_EVENT.HOST_AUDIT:
|
|
if 0 == self._wait_time:
|
|
self._wait_time = timers.get_monotonic_timestamp_in_ms()
|
|
now_ms = timers.get_monotonic_timestamp_in_ms()
|
|
secs_expired = (now_ms - self._wait_time) // 1000
|
|
# Wait 30 seconds before checking kube upgrade for first time
|
|
if 30 <= secs_expired and not self._query_inprogress:
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_kube_upgrade(self._get_kube_upgrade_callback())
|
|
return True
|
|
return False
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(AbstractKubeUpgradeStep, self).from_dict(data)
|
|
# these two attributes are not persisted
|
|
self._wait_time = 0
|
|
self._query_inprogress = False
|
|
# validation states are persisted
|
|
self._success_state = data['success_state']
|
|
self._fail_state = data['fail_state']
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the kube upgrade step as a dictionary
|
|
"""
|
|
data = super(AbstractKubeUpgradeStep, self).as_dict()
|
|
data['success_state'] = self._success_state
|
|
data['fail_state'] = self._fail_state
|
|
return data
|
|
|
|
|
|
class KubeUpgradeAbortStep(AbstractKubeUpgradeStep):
|
|
"""Kube Upgrade - Abort - Strategy Step"""
|
|
|
|
def __init__(self):
|
|
from nfv_vim import nfvi
|
|
super(KubeUpgradeAbortStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_ABORT,
|
|
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_ABORTED,
|
|
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_ABORTING_FAILED)
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Kube Upgrade - Abort - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_upgrade = response['result-data']
|
|
|
|
# Calling abort on an aborted update returns a failure so we check
|
|
if self.strategy is None:
|
|
# return success if there is no more strategy
|
|
self.stage.step_complete(strategy.STRATEGY_STEP_RESULT.SUCCESS,
|
|
"no strategy")
|
|
elif self.strategy.nfvi_kube_upgrade is None:
|
|
# return success if there is no more kube upgrade
|
|
self.stage.step_complete(strategy.STRATEGY_STEP_RESULT.SUCCESS,
|
|
"no kube upgrade")
|
|
elif self.strategy.nfvi_kube_upgrade.state == self._success_state:
|
|
self.stage.step_complete(strategy.STRATEGY_STEP_RESULT.SUCCESS,
|
|
"")
|
|
else:
|
|
# If the state does not match, the abort failed.
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result,
|
|
"Unexpected state: %s"
|
|
% self.strategy.nfvi_kube_upgrade.state)
|
|
|
|
def apply(self):
|
|
"""Kube Upgrade - Abort"""
|
|
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
nfvi.nfvi_kube_upgrade_abort(self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class KubeUpgradeStartStep(AbstractKubeUpgradeStep):
|
|
"""Kube Upgrade Start - Strategy Step"""
|
|
|
|
def __init__(self, to_version, force=False):
|
|
|
|
from nfv_vim import nfvi
|
|
|
|
super(KubeUpgradeStartStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_START,
|
|
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED,
|
|
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED,
|
|
timeout_in_secs=1800)
|
|
# next 2 attributes must be persisted through from_dict/as_dict
|
|
self._to_version = to_version
|
|
self._force = force
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
return [KubeUpgradeAbortStep()]
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(KubeUpgradeStartStep, self).from_dict(data)
|
|
self._to_version = data['to_version']
|
|
self._force = data['force']
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the kube upgrade step as a dictionary
|
|
"""
|
|
data = super(KubeUpgradeStartStep, self).as_dict()
|
|
data['to_version'] = self._to_version
|
|
data['force'] = self._force
|
|
return data
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Kube Upgrade Start - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_upgrade = response['result-data']
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube Upgrade Start"""
|
|
|
|
from nfv_vim import nfvi
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
|
|
alarm_ignore_list = ["900.401", ] # ignore the auto apply alarm
|
|
nfvi.nfvi_kube_upgrade_start(self._to_version,
|
|
self._force,
|
|
alarm_ignore_list,
|
|
self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class KubeUpgradeCleanupAbortedStep(AbstractKubeUpgradeStep):
|
|
"""Kube Upgrade Cleanup Aborted - Strategy Step"""
|
|
|
|
# todo(abailey): this class could be replaced by KubeUpgradeCleanupStep
|
|
# if it was enhanced to take an optional 'filter'
|
|
def __init__(self):
|
|
super(KubeUpgradeCleanupAbortedStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_CLEANUP_ABORTED,
|
|
None, # there is no success state for this cleanup activity
|
|
None) # there is no failure state for this cleanup activity
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Kube Upgrade Cleanup Aborted - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
# kube-upgrade-cleanup-aborted will return a result when it completes,
|
|
# so we do not want to use handle_event
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
# cleanup deletes the kube upgrade, clear it from the strategy
|
|
self.strategy.nfvi_kube_upgrade = None
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube Upgrade Cleanup Aborted"""
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
|
|
from nfv_vim import nfvi
|
|
# We only invoke this step if the state matches our filter
|
|
filter_state = nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_ABORTED
|
|
if self.strategy is not None:
|
|
if self.strategy.nfvi_kube_upgrade is not None:
|
|
if self.strategy.nfvi_kube_upgrade.state == filter_state:
|
|
DLOG.info("%s cleaning up %s" % (self._name, self.strategy.nfvi_kube_upgrade.state))
|
|
nfvi.nfvi_kube_upgrade_cleanup(self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
# All other cases, we claim success since the filter did not match
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
|
|
class KubeUpgradeCleanupStep(AbstractKubeUpgradeStep):
|
|
"""Kube Upgrade Cleanup - Strategy Step"""
|
|
|
|
def __init__(self):
|
|
super(KubeUpgradeCleanupStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_CLEANUP,
|
|
None, # there is no success state for this cleanup activity
|
|
None) # there is no failure state for this cleanup activity
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Kube Upgrade Cleanup - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
# kube-upgrade-cleanup will return a result when it completes,
|
|
# so we do not want to use handle_event
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
# cleanup deletes the kube upgrade, clear it from the strategy
|
|
self.strategy.nfvi_kube_upgrade = None
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube Upgrade Cleanup"""
|
|
|
|
from nfv_vim import nfvi
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
|
|
nfvi.nfvi_kube_upgrade_cleanup(self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class KubeUpgradeCompleteStep(AbstractKubeUpgradeStep):
|
|
"""Kube Upgrade Complete - Strategy Step"""
|
|
|
|
def __init__(self):
|
|
from nfv_vim import nfvi
|
|
super(KubeUpgradeCompleteStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_COMPLETE,
|
|
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE,
|
|
None) # there is no failure state for upgrade-complete
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
return [KubeUpgradeAbortStep()]
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Kube Upgrade Complete - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
# kube-upgrade-complete will return a result when it completes,
|
|
# so we do not want to use handle_event
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_upgrade = response['result-data']
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube Upgrade Complete """
|
|
|
|
from nfv_vim import nfvi
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
|
|
nfvi.nfvi_kube_upgrade_complete(self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class KubeUpgradeDownloadImagesStep(AbstractKubeUpgradeStep):
|
|
"""Kube Upgrade Download Images - Strategy Step"""
|
|
|
|
def __init__(self):
|
|
from nfv_vim import nfvi
|
|
super(KubeUpgradeDownloadImagesStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES,
|
|
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
|
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED,
|
|
timeout_in_secs=1800)
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
return [KubeUpgradeAbortStep()]
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Kube Upgrade Download Images - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_upgrade = response['result-data']
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube Upgrade Download Images """
|
|
|
|
from nfv_vim import nfvi
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
|
|
nfvi.nfvi_kube_upgrade_download_images(self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep):
|
|
"""Kube Upgrade Networking - Strategy Step"""
|
|
|
|
def __init__(self):
|
|
from nfv_vim import nfvi
|
|
super(KubeUpgradeNetworkingStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING,
|
|
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_NETWORKING,
|
|
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_NETWORKING_FAILED,
|
|
timeout_in_secs=900)
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
return [KubeUpgradeAbortStep()]
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Kube Upgrade Networking - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_upgrade = response['result-data']
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube Upgrade Networking"""
|
|
|
|
from nfv_vim import nfvi
|
|
DLOG.info("Step (%s) apply." % self._name)
|
|
|
|
nfvi.nfvi_kube_upgrade_networking(self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class KubeUpgradeStorageStep(AbstractKubeUpgradeStep):
|
|
"""Kube Upgrade Storage - Strategy Step"""
|
|
|
|
def __init__(self):
|
|
from nfv_vim import nfvi
|
|
super(KubeUpgradeStorageStep, self).__init__(
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE,
|
|
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_STORAGE,
|
|
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_STORAGE_FAILED,
|
|
timeout_in_secs=900)
|
|
|
|
@coroutine
|
|
def _response_callback(self):
|
|
"""Kube Upgrade Storage - Callback"""
|
|
|
|
response = (yield)
|
|
DLOG.debug("%s callback response=%s." % (self._name, response))
|
|
|
|
if response['completed']:
|
|
if self.strategy is not None:
|
|
self.strategy.nfvi_kube_upgrade = response['result-data']
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def apply(self):
|
|
"""Kube Upgrade Storage"""
|
|
|
|
from nfv_vim import nfvi
|
|
|
|
nfvi.nfvi_kube_upgrade_storage(self._response_callback())
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
|
|
|
|
class AbstractKubeHostUpgradeStep(AbstractKubeUpgradeStep):
|
|
"""Kube Upgrade Host - Abstract Strategy Step
|
|
|
|
This operation issues a host command, which updates the kube upgrade object
|
|
"""
|
|
def __init__(self,
|
|
host,
|
|
to_version,
|
|
force,
|
|
step_name,
|
|
success_state,
|
|
fail_state,
|
|
timeout_in_secs=600):
|
|
super(AbstractKubeHostUpgradeStep, self).__init__(
|
|
step_name,
|
|
success_state,
|
|
fail_state,
|
|
timeout_in_secs=timeout_in_secs)
|
|
self._to_version = to_version
|
|
self._force = force
|
|
# This class accepts only a single host
|
|
# but serializes as a list of hosts (list size of one)
|
|
self._hosts = list()
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
self._hosts.append(host)
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(AbstractKubeHostUpgradeStep, self).from_dict(data)
|
|
self._to_version = data['to_version']
|
|
self._force = data['force']
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the step as a dictionary
|
|
"""
|
|
data = super(AbstractKubeHostUpgradeStep, self).as_dict()
|
|
data['to_version'] = self._to_version
|
|
data['force'] = self._force
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
return data
|
|
|
|
|
|
class AbstractKubeHostListUpgradeStep(AbstractKubeUpgradeStep):
|
|
"""Kube Upgrade Host List - Abstract Strategy Step
|
|
|
|
This operation issues a host command, which updates the kube upgrade object
|
|
It operates on a list of hosts
|
|
Kube host operations can have intermediate (to_version) steps
|
|
"""
|
|
def __init__(self,
|
|
hosts,
|
|
to_version,
|
|
force,
|
|
step_name,
|
|
success_state,
|
|
fail_state,
|
|
timeout_in_secs=600):
|
|
super(AbstractKubeHostListUpgradeStep, self).__init__(
|
|
step_name,
|
|
success_state,
|
|
fail_state,
|
|
timeout_in_secs=timeout_in_secs)
|
|
self._to_version = to_version
|
|
self._force = force
|
|
self._hosts = hosts
|
|
self._host_names = list()
|
|
self._host_uuids = list()
|
|
for host in hosts:
|
|
self._host_names.append(host.name)
|
|
self._host_uuids.append(host.uuid)
|
|
|
|
def from_dict(self, data):
|
|
"""
|
|
Returns the step object initialized using the given dictionary
|
|
"""
|
|
super(AbstractKubeHostListUpgradeStep, self).from_dict(data)
|
|
self._to_version = data['to_version']
|
|
self._force = data['force']
|
|
self._hosts = list()
|
|
self._host_uuids = list()
|
|
self._host_names = data['entity_names']
|
|
host_table = tables.tables_get_host_table()
|
|
for host_name in self._host_names:
|
|
host = host_table.get(host_name, None)
|
|
if host is not None:
|
|
self._hosts.append(host)
|
|
self._host_uuids.append(host.uuid)
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the step as a dictionary
|
|
"""
|
|
data = super(AbstractKubeHostListUpgradeStep, self).as_dict()
|
|
data['to_version'] = self._to_version
|
|
data['force'] = self._force
|
|
data['entity_type'] = 'hosts'
|
|
data['entity_names'] = self._host_names
|
|
data['entity_uuids'] = self._host_uuids
|
|
return data
|
|
|
|
|
|
class KubeHostCordonStep(AbstractKubeHostUpgradeStep):
|
|
"""Kube Host Cordon - Strategy Step"""
|
|
|
|
def __init__(self, host, to_version, force, target_state, target_failure_state,
|
|
timeout_in_secs=600):
|
|
super(KubeHostCordonStep, self).__init__(
|
|
host,
|
|
to_version,
|
|
force,
|
|
STRATEGY_STEP_NAME.KUBE_HOST_CORDON,
|
|
target_state,
|
|
target_failure_state,
|
|
timeout_in_secs)
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
# todo(abailey): Unknown if this should include an uncordon if it fails
|
|
return [KubeUpgradeAbortStep()]
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events - does not query kube host upgrade list but
|
|
instead queries kube host upgrade directly.
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.KUBE_HOST_CORDON_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(
|
|
result,
|
|
"kube host cordon (%s) failed" % host.name)
|
|
return True
|
|
# return handle_event of parent class
|
|
return super(KubeHostCordonStep, self).handle_event(
|
|
event, event_data=event_data)
|
|
|
|
def apply(self):
|
|
"""Kube Host Cordon"""
|
|
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply to hostnames (%s)."
|
|
% (self._name, self._host_names))
|
|
host_director = directors.get_host_director()
|
|
operation = \
|
|
host_director.kube_host_cordon(self._host_names,
|
|
self._force)
|
|
return validate_operation(operation)
|
|
|
|
|
|
class KubeHostUncordonStep(AbstractKubeHostUpgradeStep):
|
|
"""Kube Host Uncordon - Strategy Step"""
|
|
|
|
def __init__(self, host, to_version, force, target_state, target_failure_state,
|
|
timeout_in_secs=600):
|
|
super(KubeHostUncordonStep, self).__init__(
|
|
host,
|
|
to_version,
|
|
force,
|
|
STRATEGY_STEP_NAME.KUBE_HOST_UNCORDON,
|
|
target_state,
|
|
target_failure_state,
|
|
timeout_in_secs)
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
# todo(abailey): Unknown if this should include a cordon if it fails
|
|
return [KubeUpgradeAbortStep()]
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events - does not query kube host upgrade list but
|
|
instead queries kube host upgrade directly.
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.KUBE_HOST_UNCORDON_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(
|
|
result,
|
|
"kube host uncordon (%s) failed" % host.name)
|
|
return True
|
|
# return handle_event of parent class
|
|
return super(KubeHostUncordonStep, self).handle_event(
|
|
event, event_data=event_data)
|
|
|
|
def apply(self):
|
|
"""Kube Host Uncordon"""
|
|
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply to hostnames (%s)."
|
|
% (self._name, self._host_names))
|
|
host_director = directors.get_host_director()
|
|
operation = \
|
|
host_director.kube_host_uncordon(self._host_names,
|
|
self._force)
|
|
return validate_operation(operation)
|
|
|
|
|
|
class KubeHostUpgradeControlPlaneStep(AbstractKubeHostUpgradeStep):
|
|
"""Kube Host Upgrade Control Plane - Strategy Step
|
|
|
|
This operation issues a host command, which updates the kube upgrade object
|
|
"""
|
|
|
|
def __init__(self, host, to_version, force, target_state, target_failure_state,
|
|
timeout_in_secs=600):
|
|
super(KubeHostUpgradeControlPlaneStep, self).__init__(
|
|
host,
|
|
to_version,
|
|
force,
|
|
STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_CONTROL_PLANE,
|
|
target_state,
|
|
target_failure_state,
|
|
timeout_in_secs)
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
# todo(abailey): Unknown if this should include an uncordon if it fails
|
|
return [KubeUpgradeAbortStep()]
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events - does not query kube host upgrade list but
|
|
instead queries kube host upgrade directly.
|
|
"""
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.KUBE_HOST_UPGRADE_CONTROL_PLANE_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(
|
|
result,
|
|
"kube host upgrade control plane (%s) failed" % host.name)
|
|
return True
|
|
# return handle_event of parent class
|
|
return super(KubeHostUpgradeControlPlaneStep, self).handle_event(
|
|
event, event_data=event_data)
|
|
|
|
def apply(self):
|
|
"""Kube Host Upgrade Control Plane"""
|
|
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply to hostnames (%s)."
|
|
% (self._name, self._host_names))
|
|
host_director = directors.get_host_director()
|
|
operation = \
|
|
host_director.kube_upgrade_hosts_control_plane(self._host_names,
|
|
self._force)
|
|
return validate_operation(operation)
|
|
|
|
|
|
class KubeHostUpgradeKubeletStep(AbstractKubeHostListUpgradeStep):
|
|
"""Kube Host Upgrade Kubelet - Strategy Step
|
|
|
|
This operation issues a host command, which indirectly updates the kube
|
|
upgrade object, however additional calls to other hosts do not change it.
|
|
"""
|
|
|
|
def __init__(self, hosts, to_version, force=True):
|
|
super(KubeHostUpgradeKubeletStep, self).__init__(
|
|
hosts,
|
|
to_version,
|
|
force,
|
|
STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_KUBELET,
|
|
None, # there is no kube upgrade success state for kubelets
|
|
None, # there is no kube upgrade failure state for kubelets
|
|
timeout_in_secs=900) # kubelet takes longer than control plane
|
|
|
|
def abort(self):
|
|
"""
|
|
Returns the abort step related to this step
|
|
"""
|
|
# todo(abailey): Unknown if this should include an uncordon if it fails
|
|
return [KubeUpgradeAbortStep()]
|
|
|
|
@coroutine
|
|
def _get_kube_host_upgrade_list_callback(self):
|
|
"""Get Kube Host Upgrade List Callback"""
|
|
|
|
from nfv_vim import nfvi
|
|
|
|
response = (yield)
|
|
DLOG.debug("(%s) callback response=%s." % (self._name, response))
|
|
|
|
self._query_inprogress = False
|
|
if response['completed']:
|
|
self.strategy.nfvi_kube_host_upgrade_list = response['result-data']
|
|
|
|
host_count = 0
|
|
match_count = 0
|
|
for host_uuid in self._host_uuids:
|
|
for k_host in self.strategy.nfvi_kube_host_upgrade_list:
|
|
if k_host.host_uuid == host_uuid:
|
|
if (k_host.kubelet_version == self._to_version and
|
|
k_host.status == nfvi.objects.v1.KUBE_HOST_UPGRADE_STATE.
|
|
KUBE_HOST_UPGRADED_KUBELET):
|
|
DLOG.info("Kubelet upgraded to version %s for host %s"
|
|
% (self._to_version, host_uuid))
|
|
match_count += 1
|
|
host_count += 1
|
|
# break out of inner loop, since uuids match
|
|
break
|
|
if host_count == len(self._host_uuids):
|
|
# this is a pointless break
|
|
break
|
|
if match_count == len(self._host_uuids):
|
|
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
|
DLOG.info("Kubelet upgrade completed")
|
|
self.stage.step_complete(result, "")
|
|
else:
|
|
# keep waiting for kubelet state to change
|
|
pass
|
|
else:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
DLOG.info("Kubelet upgrade failed")
|
|
self.stage.step_complete(result, response['reason'])
|
|
|
|
def handle_event(self, event, event_data=None):
|
|
"""
|
|
Handle Host events - queries kube host upgrade list
|
|
Override to bypass checking for kube upgrade state.
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
|
|
|
|
if event == STRATEGY_EVENT.KUBE_HOST_UPGRADE_KUBELET_FAILED:
|
|
host = event_data
|
|
if host is not None and host.name in self._host_names:
|
|
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
|
self.stage.step_complete(
|
|
result,
|
|
"kube host upgrade kubelet (%s) failed" % host.name)
|
|
return True
|
|
elif event == STRATEGY_EVENT.KUBE_HOST_UPGRADE_CHANGED:
|
|
DLOG.info("Event %s in progress" % (STRATEGY_EVENT.KUBE_HOST_UPGRADE_CHANGED))
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_kube_host_upgrade_list(
|
|
self._get_kube_host_upgrade_list_callback())
|
|
return True
|
|
elif event == STRATEGY_EVENT.HOST_AUDIT:
|
|
DLOG.info("Event %s in progress" % (STRATEGY_EVENT.HOST_AUDIT))
|
|
# Wait time not required as we have a timeout initialized
|
|
# in init method.
|
|
if not self._query_inprogress:
|
|
self._query_inprogress = True
|
|
nfvi.nfvi_get_kube_host_upgrade_list(
|
|
self._get_kube_host_upgrade_list_callback())
|
|
return True
|
|
return False
|
|
|
|
def apply(self):
|
|
"""Kube Upgrade Kubelet"""
|
|
|
|
from nfv_vim import directors
|
|
|
|
DLOG.info("Step (%s) apply to hostnames (%s)."
|
|
% (self._name, self._host_names))
|
|
host_director = directors.get_host_director()
|
|
operation = \
|
|
host_director.kube_upgrade_hosts_kubelet(self._host_names,
|
|
self._force)
|
|
if operation.is_inprogress():
|
|
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
|
elif operation.is_failed():
|
|
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
|
|
|
|
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
|
|
|
|
|
|
def strategy_step_rebuild_from_dict(data):
|
|
"""
|
|
Returns the strategy step object initialized using the given dictionary
|
|
"""
|
|
rebuild_map = {
|
|
STRATEGY_STEP_NAME.APPLY_PATCHES: ApplySwPatchesStep,
|
|
#
|
|
# kube rootca update steps
|
|
#
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_ABORT:
|
|
KubeRootcaUpdateAbortStep,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_COMPLETE:
|
|
KubeRootcaUpdateCompleteStep,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_GENERATE_CERT:
|
|
KubeRootcaUpdateGenerateCertStep,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_HOST_TRUSTBOTHCAS:
|
|
KubeRootcaUpdateHostTrustBothcasStep,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_HOST_TRUSTNEWCA:
|
|
KubeRootcaUpdateHostTrustNewcaStep,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_HOST_UPDATECERTS:
|
|
KubeRootcaUpdateHostUpdateCertsStep,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_PODS_TRUSTBOTHCAS:
|
|
KubeRootcaUpdatePodsTrustBothcasStep,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_PODS_TRUSTNEWCA:
|
|
KubeRootcaUpdatePodsTrustNewcaStep,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_START:
|
|
KubeRootcaUpdateStartStep,
|
|
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_UPLOAD_CERT:
|
|
KubeRootcaUpdateUploadCertStep,
|
|
STRATEGY_STEP_NAME.QUERY_KUBE_ROOTCA_UPDATE:
|
|
QueryKubeRootcaUpdateStep,
|
|
STRATEGY_STEP_NAME.QUERY_KUBE_ROOTCA_HOST_UPDATES:
|
|
QueryKubeRootcaHostUpdatesStep,
|
|
#
|
|
# kube upgrade steps
|
|
#
|
|
STRATEGY_STEP_NAME.KUBE_HOST_CORDON: KubeHostCordonStep,
|
|
STRATEGY_STEP_NAME.KUBE_HOST_UNCORDON: KubeHostUncordonStep,
|
|
STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_CONTROL_PLANE:
|
|
KubeHostUpgradeControlPlaneStep,
|
|
STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_KUBELET:
|
|
KubeHostUpgradeKubeletStep,
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_ABORT: KubeUpgradeAbortStep,
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_CLEANUP: KubeUpgradeCleanupStep,
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_CLEANUP_ABORTED:
|
|
KubeUpgradeCleanupAbortedStep,
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_COMPLETE: KubeUpgradeCompleteStep,
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES:
|
|
KubeUpgradeDownloadImagesStep,
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING: KubeUpgradeNetworkingStep,
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE: KubeUpgradeStorageStep,
|
|
STRATEGY_STEP_NAME.KUBE_UPGRADE_START: KubeUpgradeStartStep,
|
|
STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE: QueryKubeHostUpgradeStep,
|
|
STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE: QueryKubeUpgradeStep,
|
|
STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS: QueryKubeVersionsStep,
|
|
#
|
|
# system config update steps
|
|
#
|
|
STRATEGY_STEP_NAME.QUERY_SYSTEM_CONFIG_UPDATE_HOSTS:
|
|
QuerySystemConfigUpdateHostsStep,
|
|
STRATEGY_STEP_NAME.SYSTEM_CONFIG_UPDATE_HOSTS:
|
|
SystemConfigUpdateHostsStep,
|
|
}
|
|
obj_type = rebuild_map.get(data['name'])
|
|
if obj_type is not None:
|
|
step_obj = object.__new__(obj_type)
|
|
|
|
elif STRATEGY_STEP_NAME.SYSTEM_STABILIZE == data['name']:
|
|
step_obj = object.__new__(SystemStabilizeStep)
|
|
|
|
elif STRATEGY_STEP_NAME.UNLOCK_HOSTS == data['name']:
|
|
step_obj = object.__new__(UnlockHostsStep)
|
|
|
|
elif STRATEGY_STEP_NAME.REBOOT_HOSTS == data['name']:
|
|
step_obj = object.__new__(RebootHostsStep)
|
|
|
|
elif STRATEGY_STEP_NAME.LOCK_HOSTS == data['name']:
|
|
step_obj = object.__new__(LockHostsStep)
|
|
|
|
elif STRATEGY_STEP_NAME.SWACT_HOSTS == data['name']:
|
|
step_obj = object.__new__(SwactHostsStep)
|
|
|
|
elif STRATEGY_STEP_NAME.UPGRADE_HOSTS == data['name']:
|
|
step_obj = object.__new__(UpgradeHostsStep)
|
|
|
|
elif STRATEGY_STEP_NAME.START_UPGRADE == data['name']:
|
|
step_obj = object.__new__(UpgradeStartStep)
|
|
|
|
elif STRATEGY_STEP_NAME.ACTIVATE_UPGRADE == data['name']:
|
|
step_obj = object.__new__(UpgradeActivateStep)
|
|
|
|
elif STRATEGY_STEP_NAME.COMPLETE_UPGRADE == data['name']:
|
|
step_obj = object.__new__(UpgradeCompleteStep)
|
|
|
|
elif STRATEGY_STEP_NAME.SW_PATCH_HOSTS == data['name']:
|
|
step_obj = object.__new__(SwPatchHostsStep)
|
|
|
|
elif STRATEGY_STEP_NAME.MIGRATE_INSTANCES == data['name']:
|
|
step_obj = object.__new__(MigrateInstancesStep)
|
|
|
|
elif STRATEGY_STEP_NAME.MIGRATE_INSTANCES_FROM_HOST == data['name']:
|
|
step_obj = object.__new__(MigrateInstancesFromHostStep)
|
|
|
|
elif STRATEGY_STEP_NAME.START_INSTANCES == data['name']:
|
|
step_obj = object.__new__(StartInstancesStep)
|
|
|
|
elif STRATEGY_STEP_NAME.STOP_INSTANCES == data['name']:
|
|
step_obj = object.__new__(StopInstancesStep)
|
|
|
|
elif STRATEGY_STEP_NAME.QUERY_ALARMS == data['name']:
|
|
step_obj = object.__new__(QueryAlarmsStep)
|
|
|
|
elif STRATEGY_STEP_NAME.WAIT_DATA_SYNC == data['name']:
|
|
step_obj = object.__new__(WaitDataSyncStep)
|
|
|
|
elif STRATEGY_STEP_NAME.WAIT_ALARMS_CLEAR == data['name']:
|
|
step_obj = object.__new__(WaitAlarmsClearStep)
|
|
|
|
elif STRATEGY_STEP_NAME.QUERY_SW_PATCHES == data['name']:
|
|
step_obj = object.__new__(QuerySwPatchesStep)
|
|
|
|
elif STRATEGY_STEP_NAME.QUERY_SW_PATCH_HOSTS == data['name']:
|
|
step_obj = object.__new__(QuerySwPatchHostsStep)
|
|
|
|
elif STRATEGY_STEP_NAME.QUERY_UPGRADE == data['name']:
|
|
step_obj = object.__new__(QueryUpgradeStep)
|
|
|
|
elif STRATEGY_STEP_NAME.DISABLE_HOST_SERVICES == data['name']:
|
|
step_obj = object.__new__(DisableHostServicesStep)
|
|
|
|
elif STRATEGY_STEP_NAME.ENABLE_HOST_SERVICES == data['name']:
|
|
step_obj = object.__new__(EnableHostServicesStep)
|
|
|
|
elif STRATEGY_STEP_NAME.FW_UPDATE_HOSTS == data['name']:
|
|
step_obj = object.__new__(FwUpdateHostsStep)
|
|
|
|
elif STRATEGY_STEP_NAME.FW_UPDATE_ABORT_HOSTS == data['name']:
|
|
step_obj = object.__new__(FwUpdateAbortHostsStep)
|
|
|
|
elif STRATEGY_STEP_NAME.QUERY_FW_UPDATE_HOST == data['name']:
|
|
step_obj = object.__new__(QueryFwUpdateHostStep)
|
|
|
|
else:
|
|
step_obj = object.__new__(strategy.StrategyStep)
|
|
|
|
step_obj.from_dict(data)
|
|
return step_obj
|