Implement import-load and starting-upgrade strategy states
This commit allows the state machine to be defined through an ordered list of states. The states are mapped to by a dictionary to state handler classes. This greatly simplifies the software upgrade orch thread so that it can be refactored in the future to share functionality with other orch thread strategy types. The following states have been implemented: - 'importing load' - 'starting upgrade' - 'activating upgrade' - 'completing upgrade' The 'installing license' state now uses this design pattern. Its functionality and unit tests are now in their own files. The 'unlocking' state requires additional changes for the 'activating' and 'completing' states to pass. Change-Id: Iedc40bdea406461dd4c2ce3887a37be6ec3c999f Story: 2007403 Task: 40001 Signed-off-by: albailey <Al.Bailey@windriver.com>
This commit is contained in:
parent
bd35524bf2
commit
5e5c416a9b
|
@ -235,6 +235,11 @@ class SysinvClient(base.DriverBase):
|
|||
LOG.error("delete_load exception={}".format(e))
|
||||
raise e
|
||||
|
||||
def import_load(self, path_to_iso, path_to_sig):
|
||||
"""Import the particular software load."""
|
||||
return self.sysinv_client.load.import_load(path_to_iso=path_to_iso,
|
||||
path_to_sig=path_to_sig)
|
||||
|
||||
def get_hosts(self):
|
||||
"""Get a list of hosts."""
|
||||
return self.sysinv_client.ihost.list()
|
||||
|
@ -243,6 +248,21 @@ class SysinvClient(base.DriverBase):
|
|||
"""Get a list of upgrades."""
|
||||
return self.sysinv_client.upgrade.list()
|
||||
|
||||
def upgrade_activate(self):
|
||||
"""Invoke the API for 'system upgrade-activate', which is an update """
|
||||
patch = [{'op': 'replace',
|
||||
'path': '/state',
|
||||
'value': 'activation-requested'}, ]
|
||||
return self.sysinv_client.upgrade.update(patch)
|
||||
|
||||
def upgrade_complete(self):
|
||||
"""Invoke the API for 'system upgrade-complete', which is a delete"""
|
||||
return self.sysinv_client.upgrade.delete()
|
||||
|
||||
def upgrade_start(self, force=False):
|
||||
"""Invoke the API for 'system upgrade-start', which is a create"""
|
||||
return self.sysinv_client.upgrade.create(force)
|
||||
|
||||
def get_applications(self):
|
||||
"""Get a list of containerized applications"""
|
||||
|
||||
|
|
|
@ -108,7 +108,8 @@ STRATEGY_STATE_LOCKING_CONTROLLER = "locking controller"
|
|||
STRATEGY_STATE_UPGRADING_SIMPLEX = "upgrading simplex"
|
||||
STRATEGY_STATE_MIGRATING_DATA = "migrating data"
|
||||
STRATEGY_STATE_UNLOCKING_CONTROLLER = "unlocking controller"
|
||||
STRATEGY_STATE_ACTIVATING = "activating"
|
||||
STRATEGY_STATE_ACTIVATING_UPGRADE = "activating upgrade"
|
||||
STRATEGY_STATE_COMPLETING_UPGRADE = "completing upgrade"
|
||||
|
||||
# Subcloud deploy status states
|
||||
DEPLOY_STATE_NONE = 'not-deployed'
|
||||
|
|
|
@ -178,6 +178,10 @@ class LicenseMissingError(DCManagerException):
|
|||
message = _("License does not exist on subcloud: %(subcloud_id)s")
|
||||
|
||||
|
||||
class VaultLoadMissingError(DCManagerException):
|
||||
message = _("No matching: %(file_type) found in vault: %(vault_dir)")
|
||||
|
||||
|
||||
class StrategyStepNotFound(NotFound):
|
||||
message = _("StrategyStep with subcloud_id %(subcloud_id)s "
|
||||
"doesn't exist.")
|
||||
|
|
|
@ -28,6 +28,13 @@ class BaseState(object):
|
|||
self.get_region_name(strategy_step),
|
||||
details))
|
||||
|
||||
def info_log(self, strategy_step, details):
|
||||
LOG.info("Stage: %s, State: %s, Subcloud: %s, Details: %s"
|
||||
% (strategy_step.stage,
|
||||
strategy_step.state,
|
||||
self.get_region_name(strategy_step),
|
||||
details))
|
||||
|
||||
@staticmethod
|
||||
def get_region_name(strategy_step):
|
||||
"""Get the region name for a strategy step"""
|
||||
|
|
|
@ -5,12 +5,9 @@
|
|||
#
|
||||
import time
|
||||
|
||||
from oslo_log import log as logging
|
||||
|
||||
from dcmanager.common.consts import ADMIN_LOCKED
|
||||
from dcmanager.manager.states.base import BaseState
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
DEFAULT_MAX_QUERIES = 6
|
||||
DEFAULT_SLEEP_DURATION = 10
|
||||
|
||||
|
@ -46,7 +43,7 @@ class LockHostState(BaseState):
|
|||
if host.administrative == ADMIN_LOCKED:
|
||||
msg = "Host: %s already: %s." % (self.target_hostname,
|
||||
host.administrative)
|
||||
self.debug_log(strategy_step, msg)
|
||||
self.info_log(strategy_step, msg)
|
||||
return True
|
||||
|
||||
# Invoke the action
|
||||
|
@ -63,7 +60,7 @@ class LockHostState(BaseState):
|
|||
if host.administrative == ADMIN_LOCKED:
|
||||
msg = "Host: %s is now: %s" % (self.target_hostname,
|
||||
host.administrative)
|
||||
self.debug_log(strategy_step, msg)
|
||||
self.info_log(strategy_step, msg)
|
||||
break
|
||||
counter += 1
|
||||
if counter >= self.max_queries:
|
||||
|
|
|
@ -5,12 +5,9 @@
|
|||
#
|
||||
import time
|
||||
|
||||
from oslo_log import log as logging
|
||||
|
||||
from dcmanager.common.consts import ADMIN_UNLOCKED
|
||||
from dcmanager.manager.states.base import BaseState
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
DEFAULT_MAX_QUERIES = 6
|
||||
DEFAULT_SLEEP_DURATION = 10
|
||||
|
||||
|
@ -51,7 +48,7 @@ class UnlockHostState(BaseState):
|
|||
if host.administrative == ADMIN_UNLOCKED:
|
||||
msg = "Host: %s already: %s." % (self.target_hostname,
|
||||
host.administrative)
|
||||
self.debug_log(strategy_step, msg)
|
||||
self.info_log(strategy_step, msg)
|
||||
return True
|
||||
|
||||
# Invoke the action
|
||||
|
@ -68,7 +65,7 @@ class UnlockHostState(BaseState):
|
|||
if host.administrative == ADMIN_UNLOCKED:
|
||||
msg = "Host: %s is now: %s" % (self.target_hostname,
|
||||
host.administrative)
|
||||
self.debug_log(strategy_step, msg)
|
||||
self.info_log(strategy_step, msg)
|
||||
break
|
||||
async_counter += 1
|
||||
# check_async_counter throws exception if loops exceeded or aborted
|
||||
|
|
|
@ -3,18 +3,19 @@
|
|||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from oslo_log import log as logging
|
||||
|
||||
from dcmanager.manager.states.base import BaseState
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
ALREADY_ACTIVATING_STATES = ['activation-requested',
|
||||
'activation-failed',
|
||||
'activation-complete',
|
||||
'activating']
|
||||
|
||||
|
||||
class ActivatingState(BaseState):
|
||||
class ActivatingUpgradeState(BaseState):
|
||||
"""Upgrade state actions for activating an upgrade"""
|
||||
|
||||
def __init__(self):
|
||||
super(ActivatingState, self).__init__()
|
||||
super(ActivatingUpgradeState, self).__init__()
|
||||
|
||||
def perform_state_action(self, strategy_step):
|
||||
"""Activate an upgrade on a subcloud
|
||||
|
@ -22,9 +23,28 @@ class ActivatingState(BaseState):
|
|||
Any exceptions raised by this method set the strategy to FAILED
|
||||
Returning normally from this method set the strategy to the next step
|
||||
"""
|
||||
LOG.warning("ActivatingState has not been implemented yet.")
|
||||
# get the keystone and sysinv clients for the subcloud
|
||||
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
|
||||
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
|
||||
ks_client.session)
|
||||
upgrades = sysinv_client.get_upgrades()
|
||||
|
||||
# If there are no existing upgrades, there is nothing to activate
|
||||
if len(upgrades) == 0:
|
||||
raise Exception("No upgrades were found to activate")
|
||||
|
||||
# The list of upgrades will never contain more than one entry.
|
||||
for upgrade in upgrades:
|
||||
# Check if an existing upgrade is already activated
|
||||
if upgrade.state in ALREADY_ACTIVATING_STATES:
|
||||
self.info_log(strategy_step,
|
||||
"Already in activating state:%s" % upgrade.state)
|
||||
break
|
||||
else:
|
||||
# invoke the API 'upgrade-activate'.
|
||||
# Throws an exception on failure (no upgrade found, bad host state)
|
||||
sysinv_client.upgrade_activate()
|
||||
|
||||
# When we return from this method without throwing an exception, the
|
||||
# state machine can proceed to the next state
|
||||
LOG.warning("Faking transition to next state")
|
||||
return True
|
||||
|
|
|
@ -3,18 +3,14 @@
|
|||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from oslo_log import log as logging
|
||||
|
||||
from dcmanager.manager.states.base import BaseState
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CompletingState(BaseState):
|
||||
class CompletingUpgradeState(BaseState):
|
||||
"""Upgrade state actions for completing an upgrade"""
|
||||
|
||||
def __init__(self):
|
||||
super(CompletingState, self).__init__()
|
||||
super(CompletingUpgradeState, self).__init__()
|
||||
|
||||
def perform_state_action(self, strategy_step):
|
||||
"""Complete an upgrade on a subcloud
|
||||
|
@ -22,9 +18,24 @@ class CompletingState(BaseState):
|
|||
Any exceptions raised by this method set the strategy to FAILED
|
||||
Returning normally from this method set the strategy to the next step
|
||||
"""
|
||||
LOG.warning("CompletingState has not been implemented yet.")
|
||||
# get the keystone and sysinv clients for the subcloud
|
||||
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
|
||||
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
|
||||
ks_client.session)
|
||||
|
||||
# upgrade-complete causes the upgrade to be deleted.
|
||||
# if no upgrade exists, there is no need to call it.
|
||||
# The API should always return a list, but check for None anyways
|
||||
upgrades = sysinv_client.get_upgrades()
|
||||
if len(upgrades) == 0:
|
||||
self.info_log(strategy_step,
|
||||
"No upgrades exist. Nothing needs completing")
|
||||
return True
|
||||
|
||||
# invoke the API 'upgrade-complete'
|
||||
# This is a blocking call that raises an exception on failure.
|
||||
sysinv_client.upgrade_complete()
|
||||
|
||||
# When we return from this method without throwing an exception, the
|
||||
# state machine can proceed to the next state
|
||||
LOG.warning("Faking transition to next state")
|
||||
return True
|
||||
|
|
|
@ -1,30 +0,0 @@
|
|||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from oslo_log import log as logging
|
||||
|
||||
from dcmanager.manager.states.base import BaseState
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ImportLoadState(BaseState):
|
||||
"""Upgrade state for importing a load"""
|
||||
|
||||
def __init__(self):
|
||||
super(ImportLoadState, self).__init__()
|
||||
|
||||
def perform_state_action(self, strategy_step):
|
||||
"""Import a load on a subcloud
|
||||
|
||||
Any exceptions raised by this method set the strategy to FAILED
|
||||
Returning normally from this method set the strategy to the next step
|
||||
"""
|
||||
LOG.warning("ImportLoadState has not been implemented yet.")
|
||||
|
||||
# When we return from this method without throwing an exception, the
|
||||
# state machine can proceed to the next state
|
||||
LOG.warning("Faking transition to next state")
|
||||
return True
|
|
@ -0,0 +1,55 @@
|
|||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.manager.states.base import BaseState
|
||||
from dcmanager.manager.states.upgrade import utils
|
||||
|
||||
|
||||
class ImportingLoadState(BaseState):
|
||||
"""Upgrade state for importing a load"""
|
||||
|
||||
def __init__(self):
|
||||
super(ImportingLoadState, self).__init__()
|
||||
|
||||
def perform_state_action(self, strategy_step):
|
||||
"""Import a load on a subcloud
|
||||
|
||||
Any exceptions raised by this method set the strategy to FAILED
|
||||
Returning normally from this method set the strategy to the next step
|
||||
"""
|
||||
# determine the version of the system controller in region one
|
||||
local_ks_client = self.get_keystone_client()
|
||||
local_sysinv_client = \
|
||||
self.get_sysinv_client(consts.DEFAULT_REGION_NAME,
|
||||
local_ks_client.session)
|
||||
target_version = local_sysinv_client.get_system().software_version
|
||||
|
||||
# get the keystone and sysinv clients for the subcloud
|
||||
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
|
||||
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
|
||||
ks_client.session)
|
||||
|
||||
# Check if the load is already imported by checking what the version is
|
||||
current_loads = sysinv_client.get_loads()
|
||||
for load in current_loads:
|
||||
if load.software_version == target_version:
|
||||
self.info_log(strategy_step,
|
||||
"Load:%s already found" % target_version)
|
||||
return True
|
||||
|
||||
# If we are here, the load needs to be imported
|
||||
# ISO and SIG files are found in the vault under a version directory
|
||||
iso_path, sig_path = utils.get_vault_load_files(target_version)
|
||||
|
||||
# Call the API.
|
||||
imported_load = sysinv_client.import_load(iso_path, sig_path)
|
||||
new_load = imported_load.get('new_load', {})
|
||||
if new_load.get('software_version') != target_version:
|
||||
raise Exception("The imported load was not the expected version")
|
||||
|
||||
# When we return from this method without throwing an exception, the
|
||||
# state machine can proceed to the next state
|
||||
return True
|
|
@ -0,0 +1,86 @@
|
|||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.common import exceptions
|
||||
from dcmanager.manager.states.base import BaseState
|
||||
|
||||
# When a license is not installed, this will be part of the API error string
|
||||
LICENSE_FILE_NOT_FOUND_SUBSTRING = "License file not found"
|
||||
|
||||
|
||||
class InstallingLicenseState(BaseState):
|
||||
"""Upgrade state action for installing a license"""
|
||||
|
||||
def __init__(self):
|
||||
super(InstallingLicenseState, self).__init__()
|
||||
|
||||
@staticmethod
|
||||
def license_up_to_date(target_license, existing_license):
|
||||
return target_license == existing_license
|
||||
|
||||
def perform_state_action(self, strategy_step):
|
||||
"""Install the License for a software upgrade in this subcloud
|
||||
|
||||
Any exceptions raised by this method set the strategy to FAILED
|
||||
Returning normally from this method set the strategy to the next step
|
||||
"""
|
||||
|
||||
# check if the the system controller has a license
|
||||
local_ks_client = self.get_keystone_client()
|
||||
local_sysinv_client = \
|
||||
self.get_sysinv_client(consts.DEFAULT_REGION_NAME,
|
||||
local_ks_client.session)
|
||||
system_controller_license = local_sysinv_client.get_license()
|
||||
# get_license returns a dictionary with keys: content and error
|
||||
# 'content' can be an empty string in success or failure case.
|
||||
# 'error' is an empty string only in success case.
|
||||
target_license = system_controller_license.get('content')
|
||||
target_error = system_controller_license.get('error')
|
||||
|
||||
# If the system controller does not have a license, do not attempt
|
||||
# to install licenses on subclouds, simply proceed to the next stage
|
||||
if len(target_error) != 0:
|
||||
if LICENSE_FILE_NOT_FOUND_SUBSTRING in target_error:
|
||||
self.info_log(strategy_step,
|
||||
"System Controller License missing: %s."
|
||||
% target_error)
|
||||
return True
|
||||
else:
|
||||
# An unexpected error occurred querying the license
|
||||
raise exceptions.LicenseInstallError(
|
||||
subcloud_id=consts.SYSTEM_CONTROLLER_NAME)
|
||||
|
||||
# retrieve the keystone session for the subcloud and query its license
|
||||
subcloud_ks_client = \
|
||||
self.get_keystone_client(strategy_step.subcloud.name)
|
||||
subcloud_sysinv_client = \
|
||||
self.get_sysinv_client(strategy_step.subcloud.name,
|
||||
subcloud_ks_client.session)
|
||||
subcloud_license_response = subcloud_sysinv_client.get_license()
|
||||
subcloud_license = subcloud_license_response.get('content')
|
||||
subcloud_error = subcloud_license_response.get('error')
|
||||
|
||||
# Skip license install if the license is already up to date
|
||||
# If there was not an error, there might be a license
|
||||
if len(subcloud_error) == 0:
|
||||
if self.license_up_to_date(target_license, subcloud_license):
|
||||
self.info_log(strategy_step, "License up to date.")
|
||||
return True
|
||||
else:
|
||||
self.debug_log(strategy_step, "License mismatch. Updating.")
|
||||
else:
|
||||
self.debug_log(strategy_step, "License missing. Installing.")
|
||||
|
||||
# Install the license
|
||||
install_rc = subcloud_sysinv_client.install_license(target_license)
|
||||
install_error = install_rc.get('error')
|
||||
if len(install_error) != 0:
|
||||
raise exceptions.LicenseInstallError(
|
||||
subcloud_id=strategy_step.subcloud_id)
|
||||
|
||||
# The license has been successfully installed. Move to the next stage
|
||||
self.info_log(strategy_step, "License installed.")
|
||||
return True
|
|
@ -3,18 +3,15 @@
|
|||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from oslo_log import log as logging
|
||||
|
||||
from dcmanager.manager.states.base import BaseState
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StartingUpgradeState(BaseState):
|
||||
"""Upgrade state for starting an upgrade on a subcloud"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, force=False):
|
||||
super(StartingUpgradeState, self).__init__()
|
||||
self.force = force
|
||||
|
||||
def perform_state_action(self, strategy_step):
|
||||
"""Start an upgrade on a subcloud
|
||||
|
@ -22,9 +19,24 @@ class StartingUpgradeState(BaseState):
|
|||
Any exceptions raised by this method set the strategy to FAILED
|
||||
Returning normally from this method set the strategy to the next step
|
||||
"""
|
||||
LOG.warning("StartingUpgradeState has not been implemented yet.")
|
||||
# get the keystone and sysinv clients for the subcloud
|
||||
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
|
||||
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
|
||||
ks_client.session)
|
||||
|
||||
# Check if an existing upgrade is already in progress.
|
||||
# The list of upgrades will never contain more than one entry.
|
||||
upgrades = sysinv_client.get_upgrades()
|
||||
if upgrades is not None and len(upgrades) > 0:
|
||||
for upgrade in upgrades:
|
||||
# If a previous upgrade exists (even one that failed) skip
|
||||
self.info_log(strategy_step,
|
||||
"An upgrade already exists: %s" % upgrade)
|
||||
else:
|
||||
# invoke the API 'upgrade-start'.
|
||||
# This call is synchronous and throws an exception on failure.
|
||||
sysinv_client.upgrade_start(self.force)
|
||||
|
||||
# When we return from this method without throwing an exception, the
|
||||
# state machine can proceed to the next state
|
||||
LOG.warning("Faking transition to next state")
|
||||
return True
|
||||
|
|
|
@ -0,0 +1,38 @@
|
|||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
import os
|
||||
|
||||
from dcmanager.common.exceptions import VaultLoadMissingError
|
||||
|
||||
VAULT_LOADS_PATH = '/opt/dc-vault/loads'
|
||||
|
||||
|
||||
def get_vault_load_files(target_version):
|
||||
"""Return a tuple for the ISO and SIG for this load version from the vault.
|
||||
|
||||
The files can be imported to the vault using any name, but must end
|
||||
in 'iso' or 'sig'.
|
||||
: param target_version: The software version to search under the vault
|
||||
"""
|
||||
vault_dir = "{}/{}/".format(VAULT_LOADS_PATH, target_version)
|
||||
|
||||
matching_iso = None
|
||||
matching_sig = None
|
||||
for a_file in os.listdir(vault_dir):
|
||||
if a_file.lower().endswith(".iso"):
|
||||
matching_iso = os.path.join(vault_dir, a_file)
|
||||
continue
|
||||
elif a_file.lower().endswith(".sig"):
|
||||
matching_sig = os.path.join(vault_dir, a_file)
|
||||
continue
|
||||
# If no .iso or .sig is found, raise an exception
|
||||
if matching_iso is None:
|
||||
raise VaultLoadMissingError(file_type='.iso', vault_dir=vault_dir)
|
||||
if matching_sig is None:
|
||||
raise VaultLoadMissingError(file_type='.sig', vault_dir=vault_dir)
|
||||
|
||||
# return the iso and sig for this load
|
||||
return (matching_iso, matching_sig)
|
|
@ -25,19 +25,52 @@ import time
|
|||
|
||||
from oslo_log import log as logging
|
||||
|
||||
from dccommon.drivers.openstack.sdk_platform import OpenStackDriver
|
||||
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
|
||||
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.common import context
|
||||
from dcmanager.common import exceptions
|
||||
from dcmanager.common import scheduler
|
||||
from dcmanager.db import api as db_api
|
||||
from dcmanager.manager.states.lock_host import LockHostState
|
||||
from dcmanager.manager.states.unlock_host import UnlockHostState
|
||||
from dcmanager.manager.states.upgrade.activating import ActivatingUpgradeState
|
||||
from dcmanager.manager.states.upgrade.completing import CompletingUpgradeState
|
||||
from dcmanager.manager.states.upgrade.importing_load import ImportingLoadState
|
||||
from dcmanager.manager.states.upgrade.installing_license \
|
||||
import InstallingLicenseState
|
||||
from dcmanager.manager.states.upgrade.migrating_data \
|
||||
import MigratingDataState
|
||||
from dcmanager.manager.states.upgrade.starting_upgrade \
|
||||
import StartingUpgradeState
|
||||
from dcmanager.manager.states.upgrade.upgrading_simplex \
|
||||
import UpgradingSimplexState
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
# When a license is not installed, this will be part of the API error string
|
||||
LICENSE_FILE_NOT_FOUND_SUBSTRING = "License file not found"
|
||||
# The state machine transition order for an APPLY
|
||||
ORDERED_STATES = [
|
||||
consts.STRATEGY_STATE_INSTALLING_LICENSE,
|
||||
consts.STRATEGY_STATE_IMPORTING_LOAD,
|
||||
consts.STRATEGY_STATE_STARTING_UPGRADE,
|
||||
consts.STRATEGY_STATE_LOCKING_CONTROLLER,
|
||||
consts.STRATEGY_STATE_UPGRADING_SIMPLEX,
|
||||
consts.STRATEGY_STATE_MIGRATING_DATA,
|
||||
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER,
|
||||
consts.STRATEGY_STATE_ACTIVATING_UPGRADE,
|
||||
consts.STRATEGY_STATE_COMPLETING_UPGRADE,
|
||||
]
|
||||
|
||||
# every state in ORDERED_STATES should have an operator
|
||||
STATE_OPERATORS = {
|
||||
consts.STRATEGY_STATE_INSTALLING_LICENSE: InstallingLicenseState,
|
||||
consts.STRATEGY_STATE_IMPORTING_LOAD: ImportingLoadState,
|
||||
consts.STRATEGY_STATE_STARTING_UPGRADE: StartingUpgradeState,
|
||||
consts.STRATEGY_STATE_LOCKING_CONTROLLER: LockHostState,
|
||||
consts.STRATEGY_STATE_UPGRADING_SIMPLEX: UpgradingSimplexState,
|
||||
consts.STRATEGY_STATE_MIGRATING_DATA: MigratingDataState,
|
||||
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER: UnlockHostState,
|
||||
consts.STRATEGY_STATE_ACTIVATING_UPGRADE: ActivatingUpgradeState,
|
||||
consts.STRATEGY_STATE_COMPLETING_UPGRADE: CompletingUpgradeState,
|
||||
}
|
||||
|
||||
|
||||
class SwUpgradeOrchThread(threading.Thread):
|
||||
|
@ -72,7 +105,7 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||
self.subcloud_workers = dict()
|
||||
|
||||
# When an upgrade is initiated, this is the first state
|
||||
self.starting_state = consts.STRATEGY_STATE_INSTALLING_LICENSE
|
||||
self.starting_state = ORDERED_STATES[0]
|
||||
|
||||
def stopped(self):
|
||||
return self._stop.isSet()
|
||||
|
@ -87,22 +120,6 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||
self.thread_group_manager.stop()
|
||||
LOG.info("SwUpgradeOrchThread Stopped")
|
||||
|
||||
@staticmethod
|
||||
def get_ks_client(region_name=consts.DEFAULT_REGION_NAME):
|
||||
"""This will get a cached keystone client (and token)"""
|
||||
try:
|
||||
os_client = OpenStackDriver(
|
||||
region_name=region_name,
|
||||
region_clients=None)
|
||||
return os_client.keystone_client
|
||||
except Exception:
|
||||
LOG.warn('Failure initializing KeystoneClient')
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def get_sysinv_client(region_name, session):
|
||||
return SysinvClient(region_name, session)
|
||||
|
||||
@staticmethod
|
||||
def get_region_name(strategy_step):
|
||||
"""Get the region name for a strategy step"""
|
||||
|
@ -121,8 +138,22 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||
return details
|
||||
|
||||
@staticmethod
|
||||
def license_up_to_date(target_license, existing_license):
|
||||
return target_license == existing_license
|
||||
def determine_state_operator(strategy_step):
|
||||
"""Return the state operator for the current state"""
|
||||
state_operator = STATE_OPERATORS.get(strategy_step.state)
|
||||
# instantiate and return the state_operator class
|
||||
return state_operator()
|
||||
|
||||
@staticmethod
|
||||
def determine_next_state(strategy_step):
|
||||
"""Return next state for the strategy step based on current state."""
|
||||
# todo(abailey): next_state may differ for AIO, STD, etc.. subclouds
|
||||
next_index = ORDERED_STATES.index(strategy_step.state) + 1
|
||||
if next_index < len(ORDERED_STATES):
|
||||
next_state = ORDERED_STATES[next_index]
|
||||
else:
|
||||
next_state = consts.STRATEGY_STATE_COMPLETE
|
||||
return next_state
|
||||
|
||||
def strategy_step_update(self, subcloud_id, state=None, details=None):
|
||||
"""Update the strategy step in the DB
|
||||
|
@ -137,13 +168,13 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||
consts.STRATEGY_STATE_ABORTED,
|
||||
consts.STRATEGY_STATE_FAILED]:
|
||||
finished_at = datetime.datetime.now()
|
||||
db_api.strategy_step_update(
|
||||
self.context,
|
||||
subcloud_id,
|
||||
state=state,
|
||||
details=details,
|
||||
started_at=started_at,
|
||||
finished_at=finished_at)
|
||||
# Return the updated object, in case we need to use its updated values
|
||||
return db_api.strategy_step_update(self.context,
|
||||
subcloud_id,
|
||||
state=state,
|
||||
details=details,
|
||||
started_at=started_at,
|
||||
finished_at=finished_at)
|
||||
|
||||
def upgrade_orch(self):
|
||||
while not self.stopped():
|
||||
|
@ -288,216 +319,23 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||
continue
|
||||
|
||||
# We are just getting started, enter the first state
|
||||
self.strategy_step_update(
|
||||
# Use the updated value for calling process_upgrade_step
|
||||
strategy_step = self.strategy_step_update(
|
||||
strategy_step.subcloud_id,
|
||||
state=consts.STRATEGY_STATE_INSTALLING_LICENSE)
|
||||
|
||||
# Initial step should log an error if a greenthread exists
|
||||
# All other steps should not.
|
||||
state=self.starting_state)
|
||||
# Starting state should log an error if greenthread exists
|
||||
self.process_upgrade_step(region,
|
||||
strategy_step,
|
||||
self.install_subcloud_license,
|
||||
log_error=True)
|
||||
# todo(abailey): state and their method invoked can be managed
|
||||
# using a dictionary to make this more maintainable.
|
||||
elif strategy_step.state == \
|
||||
consts.STRATEGY_STATE_INSTALLING_LICENSE:
|
||||
self.process_upgrade_step(region,
|
||||
strategy_step,
|
||||
self.install_subcloud_license,
|
||||
log_error=False)
|
||||
elif strategy_step.state == \
|
||||
consts.STRATEGY_STATE_IMPORTING_LOAD:
|
||||
self.process_upgrade_step(region,
|
||||
strategy_step,
|
||||
self.import_subcloud_load,
|
||||
log_error=False)
|
||||
elif strategy_step.state == \
|
||||
consts.STRATEGY_STATE_STARTING_UPGRADE:
|
||||
self.process_upgrade_step(region,
|
||||
strategy_step,
|
||||
self.start_subcloud_upgrade,
|
||||
log_error=False)
|
||||
elif strategy_step.state == \
|
||||
consts.STRATEGY_STATE_LOCKING_CONTROLLER:
|
||||
self.process_upgrade_step(region,
|
||||
strategy_step,
|
||||
self.lock_subcloud_controller,
|
||||
log_error=False)
|
||||
elif strategy_step.state == \
|
||||
consts.STRATEGY_STATE_UPGRADING_SIMPLEX:
|
||||
self.process_upgrade_step(region,
|
||||
strategy_step,
|
||||
self.upgrade_subcloud_simplex,
|
||||
log_error=False)
|
||||
elif strategy_step.state == \
|
||||
consts.STRATEGY_STATE_MIGRATING_DATA:
|
||||
self.process_upgrade_step(region,
|
||||
strategy_step,
|
||||
self.migrate_subcloud_data,
|
||||
log_error=False)
|
||||
elif strategy_step.state == \
|
||||
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER:
|
||||
self.process_upgrade_step(region,
|
||||
strategy_step,
|
||||
self.unlock_subcloud_controller,
|
||||
log_error=False)
|
||||
elif strategy_step.state == \
|
||||
consts.STRATEGY_STATE_ACTIVATING:
|
||||
self.process_upgrade_step(region,
|
||||
strategy_step,
|
||||
self.activate_subcloud,
|
||||
log_error=False)
|
||||
# todo(abailey): Add calls to self.process_upgrade_step
|
||||
# for each additional state, with the appropriate thread
|
||||
# method called.
|
||||
else:
|
||||
LOG.error("Unimplemented state %s" % strategy_step.state)
|
||||
self.strategy_step_update(
|
||||
strategy_step.subcloud_id,
|
||||
state=consts.STRATEGY_STATE_FAILED,
|
||||
details=("Upgrade state not implemented: %s"
|
||||
% strategy_step.state))
|
||||
self.process_upgrade_step(region,
|
||||
strategy_step,
|
||||
log_error=False)
|
||||
|
||||
if self.stopped():
|
||||
LOG.info("Exiting because task is stopped")
|
||||
return
|
||||
|
||||
def process_upgrade_step(self,
|
||||
region,
|
||||
strategy_step,
|
||||
upgrade_thread_method,
|
||||
log_error=False):
|
||||
if region in self.subcloud_workers:
|
||||
# A worker already exists. Let it finish whatever it was doing.
|
||||
if log_error:
|
||||
LOG.error("Worker should not exist for %s." % region)
|
||||
else:
|
||||
LOG.debug("Update worker exists for %s." % region)
|
||||
else:
|
||||
# Create a greenthread to start processing the upgrade for the
|
||||
# subcloud and invoke the specified upgrade_thread_method
|
||||
self.subcloud_workers[region] = \
|
||||
self.thread_group_manager.start(upgrade_thread_method,
|
||||
strategy_step)
|
||||
|
||||
def install_subcloud_license(self, strategy_step):
|
||||
"""Install the license for the upgrade in this subcloud
|
||||
|
||||
Removes the worker reference after the operation is complete.
|
||||
"""
|
||||
|
||||
try:
|
||||
LOG.info("Stage: %s for subcloud %s"
|
||||
% (strategy_step.stage,
|
||||
self.get_region_name(strategy_step)))
|
||||
self.do_install_subcloud_license(strategy_step)
|
||||
except Exception:
|
||||
# Catch ALL exceptions and set the strategy to failed
|
||||
LOG.exception("Install license failed for %s"
|
||||
% self.get_region_name(strategy_step))
|
||||
self.strategy_step_update(strategy_step.subcloud_id,
|
||||
state=consts.STRATEGY_STATE_FAILED,
|
||||
details=("Install license failed"))
|
||||
finally:
|
||||
# The worker is done.
|
||||
region = self.get_region_name(strategy_step)
|
||||
if region in self.subcloud_workers:
|
||||
del self.subcloud_workers[region]
|
||||
|
||||
def do_install_subcloud_license(self, strategy_step):
|
||||
"""Install the License for a software upgrade in this subcloud"""
|
||||
|
||||
# Note: no need to catch exceptions in this method.
|
||||
|
||||
# next_state is the next state that the strategy will use on
|
||||
# successful completion of this state
|
||||
next_state = consts.STRATEGY_STATE_IMPORTING_LOAD
|
||||
|
||||
# We check the system controller license for system controller and
|
||||
# subclouds
|
||||
local_ks_client = self.get_ks_client()
|
||||
local_sysinv_client = \
|
||||
self.get_sysinv_client(consts.DEFAULT_REGION_NAME,
|
||||
local_ks_client.session)
|
||||
system_controller_license = local_sysinv_client.get_license()
|
||||
# get_license returns a dictionary with keys: content and error
|
||||
# 'content' can be an empty string in success or failure case.
|
||||
# 'error' is an empty string only in success case.
|
||||
target_license = system_controller_license.get('content')
|
||||
target_error = system_controller_license.get('error')
|
||||
|
||||
# If the system controller does not have a license, do not attempt
|
||||
# to install licenses on subclouds, and simply proceed to the next stage
|
||||
if len(target_error) != 0:
|
||||
if LICENSE_FILE_NOT_FOUND_SUBSTRING in target_error:
|
||||
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
|
||||
"System Controller License missing: %s."
|
||||
% (strategy_step.stage,
|
||||
self.get_region_name(strategy_step),
|
||||
target_error))
|
||||
self.strategy_step_update(strategy_step.subcloud_id,
|
||||
state=next_state)
|
||||
return
|
||||
else:
|
||||
# An unexpected API error was returned. Fail this stage.
|
||||
LOG.warning("Stage:<%s>, Subcloud:<%s>. "
|
||||
"System Controller License query failed: %s."
|
||||
% (strategy_step.stage,
|
||||
self.get_region_name(strategy_step),
|
||||
target_error))
|
||||
raise exceptions.LicenseMissingError(
|
||||
subcloud_id=consts.SYSTEM_CONTROLLER_NAME)
|
||||
|
||||
# retrieve the keystone session for the subcloud and query its license
|
||||
subcloud_ks_client = self.get_ks_client(strategy_step.subcloud.name)
|
||||
subcloud_sysinv_client = \
|
||||
self.get_sysinv_client(strategy_step.subcloud.name,
|
||||
subcloud_ks_client.session)
|
||||
subcloud_license_response = subcloud_sysinv_client.get_license()
|
||||
subcloud_license = subcloud_license_response.get('content')
|
||||
subcloud_error = subcloud_license_response.get('error')
|
||||
|
||||
# Skip license install if the license is already up to date
|
||||
# If there was not an error, there might be a license
|
||||
if len(subcloud_error) == 0:
|
||||
if self.license_up_to_date(target_license, subcloud_license):
|
||||
LOG.debug("Stage:<%s>, Subcloud:<%s>. License up to date."
|
||||
% (strategy_step.stage,
|
||||
self.get_region_name(strategy_step)))
|
||||
self.strategy_step_update(strategy_step.subcloud_id,
|
||||
state=next_state)
|
||||
return
|
||||
else:
|
||||
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
|
||||
"License mismatch. Updating."
|
||||
% (strategy_step.stage,
|
||||
self.get_region_name(strategy_step)))
|
||||
else:
|
||||
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
|
||||
"License missing. Installing."
|
||||
% (strategy_step.stage,
|
||||
self.get_region_name(strategy_step)))
|
||||
|
||||
# Install the license
|
||||
install_rc = subcloud_sysinv_client.install_license(target_license)
|
||||
install_error = install_rc.get('error')
|
||||
if len(install_error) != 0:
|
||||
LOG.warning("Stage:<%s>, Subcloud:<%s>. "
|
||||
"License install failed:<%s>."
|
||||
% (strategy_step.stage,
|
||||
self.get_region_name(strategy_step),
|
||||
install_error))
|
||||
raise exceptions.LicenseInstallError(
|
||||
subcloud_id=strategy_step.subcloud_id)
|
||||
|
||||
# The license has been successfully installed. Move to the next stage
|
||||
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
|
||||
"License installed."
|
||||
% (strategy_step.stage,
|
||||
self.get_region_name(strategy_step)))
|
||||
self.strategy_step_update(strategy_step.subcloud_id, state=next_state)
|
||||
|
||||
def abort(self, sw_update_strategy):
|
||||
"""Abort an upgrade strategy"""
|
||||
|
||||
|
@ -536,15 +374,33 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||
LOG.exception(e)
|
||||
raise e
|
||||
|
||||
def perform_state_action(self, strategy_step, state_operator, next_state):
|
||||
def process_upgrade_step(self, region, strategy_step, log_error=False):
|
||||
"""manage the green thread for calling perform_state_action"""
|
||||
if region in self.subcloud_workers:
|
||||
# A worker already exists. Let it finish whatever it was doing.
|
||||
if log_error:
|
||||
LOG.error("Worker should not exist for %s." % region)
|
||||
else:
|
||||
LOG.debug("Update worker exists for %s." % region)
|
||||
else:
|
||||
# Create a greenthread to start processing the upgrade for the
|
||||
# subcloud and invoke the specified upgrade_thread_method
|
||||
self.subcloud_workers[region] = \
|
||||
self.thread_group_manager.start(self.perform_state_action,
|
||||
strategy_step)
|
||||
|
||||
def perform_state_action(self, strategy_step):
|
||||
"""Extensible state handler for processing and transitioning states """
|
||||
try:
|
||||
LOG.info("Stage: %s, State: %s, Subcloud: %s"
|
||||
% (strategy_step.stage,
|
||||
strategy_step.state,
|
||||
self.get_region_name(strategy_step)))
|
||||
# Instantiate the state operator and perform the state actions
|
||||
state_operator = self.determine_state_operator(strategy_step)
|
||||
state_operator.perform_state_action(strategy_step)
|
||||
# If we get here without an exception raised, proceed to next state
|
||||
next_state = self.determine_next_state(strategy_step)
|
||||
self.strategy_step_update(strategy_step.subcloud_id,
|
||||
state=next_state)
|
||||
except Exception as e:
|
||||
|
@ -562,47 +418,3 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||
region = self.get_region_name(strategy_step)
|
||||
if region in self.subcloud_workers:
|
||||
del self.subcloud_workers[region]
|
||||
|
||||
# todo(abailey): convert license install to the same pattern as the other states
|
||||
|
||||
def import_subcloud_load(self, strategy_step):
|
||||
from dcmanager.manager.states.upgrade.import_load import ImportLoadState
|
||||
self.perform_state_action(strategy_step,
|
||||
ImportLoadState(),
|
||||
consts.STRATEGY_STATE_STARTING_UPGRADE)
|
||||
|
||||
def start_subcloud_upgrade(self, strategy_step):
|
||||
from dcmanager.manager.states.upgrade.starting_upgrade import StartingUpgradeState
|
||||
self.perform_state_action(strategy_step,
|
||||
StartingUpgradeState(),
|
||||
consts.STRATEGY_STATE_LOCKING_CONTROLLER)
|
||||
|
||||
def lock_subcloud_controller(self, strategy_step):
|
||||
from dcmanager.manager.states.lock_host import LockHostState
|
||||
self.perform_state_action(strategy_step,
|
||||
LockHostState(),
|
||||
consts.STRATEGY_STATE_UPGRADING_SIMPLEX)
|
||||
|
||||
def upgrade_subcloud_simplex(self, strategy_step):
|
||||
from dcmanager.manager.states.upgrade.upgrading_simplex import UpgradingSimplexState
|
||||
self.perform_state_action(strategy_step,
|
||||
UpgradingSimplexState(),
|
||||
consts.STRATEGY_STATE_MIGRATING_DATA)
|
||||
|
||||
def migrate_subcloud_data(self, strategy_step):
|
||||
from dcmanager.manager.states.upgrade.migrating_data import MigratingDataState
|
||||
self.perform_state_action(strategy_step,
|
||||
MigratingDataState(),
|
||||
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER)
|
||||
|
||||
def unlock_subcloud_controller(self, strategy_step):
|
||||
from dcmanager.manager.states.unlock_host import UnlockHostState
|
||||
self.perform_state_action(strategy_step,
|
||||
UnlockHostState(),
|
||||
consts.STRATEGY_STATE_ACTIVATING)
|
||||
|
||||
def activate_subcloud(self, strategy_step):
|
||||
from dcmanager.manager.states.upgrade.activating import ActivatingState
|
||||
self.perform_state_action(strategy_step,
|
||||
ActivatingState(),
|
||||
consts.STRATEGY_STATE_COMPLETE)
|
||||
|
|
|
@ -44,7 +44,7 @@ SUBCLOUD_SAMPLE_DATA_0 = [
|
|||
"subcloud-4", # name
|
||||
"demo subcloud", # description
|
||||
"Ottawa-Lab-Aisle_3-Rack_C", # location
|
||||
"20.01", # software-version
|
||||
"12.34", # software-version
|
||||
"managed", # management-state
|
||||
"online", # availability-status
|
||||
"fd01:3::0/64", # management_subnet
|
||||
|
|
|
@ -56,7 +56,7 @@ FAKE_SUBCLOUD_DATA = {"name": "subcloud1",
|
|||
|
||||
FAKE_SUBCLOUD_INSTALL_VALUES = {
|
||||
"image": "http://192.168.101.2:8080/iso/bootimage.iso",
|
||||
"software_version": "20.01",
|
||||
"software_version": "12.34",
|
||||
"bootstrap_interface": "eno1",
|
||||
"bootstrap_address": "128.224.151.183",
|
||||
"bootstrap_address_prefix": 23,
|
||||
|
|
|
@ -0,0 +1,91 @@
|
|||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
import mock
|
||||
|
||||
from dcmanager.common import consts
|
||||
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||
import TestSwUpgradeState
|
||||
|
||||
VALID_UPGRADE = FakeUpgrade(state='imported')
|
||||
ACTIVATING_UPGRADE = FakeUpgrade(state='activation-requested')
|
||||
ALREADY_ACTIVATED_UPGRADE = FakeUpgrade(state='activation-complete')
|
||||
|
||||
|
||||
class TestSwUpgradeActivatingStage(TestSwUpgradeState):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSwUpgradeActivatingStage, self).setUp()
|
||||
|
||||
# next state after activating an upgrade is 'completing'
|
||||
self.on_success_state = consts.STRATEGY_STATE_COMPLETING_UPGRADE
|
||||
|
||||
# Add the strategy_step state being processed by this unit test
|
||||
self.strategy_step = \
|
||||
self.setup_strategy_step(consts.STRATEGY_STATE_ACTIVATING_UPGRADE)
|
||||
|
||||
# Add mock API endpoints for sysinv client calls invcked by this state
|
||||
self.sysinv_client.upgrade_activate = mock.MagicMock()
|
||||
self.sysinv_client.get_upgrades = mock.MagicMock()
|
||||
|
||||
def test_upgrade_subcloud_activating_upgrade_failure(self):
|
||||
"""Test the activating upgrade API call fails."""
|
||||
|
||||
# upgrade_activate will only be called if an appropriate upgrade exists
|
||||
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
|
||||
|
||||
# API call raises an exception when it is rejected
|
||||
self.sysinv_client.upgrade_activate.side_effect = \
|
||||
Exception("upgrade activate failed for some reason")
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the expected API call was invoked
|
||||
self.sysinv_client.upgrade_activate.assert_called()
|
||||
|
||||
# Verify the state moves to 'failed'
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
def test_upgrade_subcloud_activating_upgrade_success(self):
|
||||
"""Test the activating upgrade step succeeds."""
|
||||
|
||||
# upgrade_activate will only be called if an appropriate upgrade exists
|
||||
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
|
||||
|
||||
# API call will not raise an exception, and will return an upgrade
|
||||
self.sysinv_client.upgrade_activate.return_value = ACTIVATING_UPGRADE
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the API cvall was invoked
|
||||
self.sysinv_client.upgrade_activate.assert_called()
|
||||
|
||||
# On success, the state should be updated to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
||||
|
||||
def test_upgrade_subcloud_activating_upgrade_skip_already_activated(self):
|
||||
"""Test the activating upgrade step skipped if already activated."""
|
||||
|
||||
# upgrade_activate will only be called if an appropriate upgrade exists
|
||||
self.sysinv_client.get_upgrades.return_value = \
|
||||
[ALREADY_ACTIVATED_UPGRADE, ]
|
||||
|
||||
# API call will not be invoked, so no need to mock it
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# upgrade is already in one of the activating states so skip activating
|
||||
self.sysinv_client.upgrade_activate.assert_not_called()
|
||||
|
||||
# On success, the state is set to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
|
@ -4,14 +4,15 @@
|
|||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
import mock
|
||||
import uuid
|
||||
|
||||
from dcmanager.manager.states.base import BaseState
|
||||
from sysinv.common import constants as sysinv_constants
|
||||
|
||||
from dcmanager.tests.unit.manager.test_sw_upgrade import TestSwUpgrade
|
||||
|
||||
CURRENT_LOAD = '20.01'
|
||||
UPDATED_LOAD = '20.06'
|
||||
PREVIOUS_VERSION = '12.34'
|
||||
UPGRADED_VERSION = '56.78'
|
||||
|
||||
|
||||
class FakeKeystoneClient(object):
|
||||
|
@ -19,6 +20,47 @@ class FakeKeystoneClient(object):
|
|||
self.session = mock.MagicMock()
|
||||
|
||||
|
||||
class FakeLoad(object):
|
||||
def __init__(self,
|
||||
obj_id,
|
||||
compatible_version='N/A',
|
||||
required_patches='N/A',
|
||||
software_version=PREVIOUS_VERSION,
|
||||
state='active',
|
||||
created_at=None,
|
||||
updated_at=None):
|
||||
self.id = obj_id
|
||||
self.uuid = uuid.uuid4()
|
||||
self.required_patches = required_patches
|
||||
self.software_version = software_version
|
||||
self.state = state
|
||||
self.created_at = created_at
|
||||
self.updated_at = updated_at
|
||||
|
||||
|
||||
class FakeSystem(object):
|
||||
def __init__(self,
|
||||
obj_id=1,
|
||||
software_version=UPGRADED_VERSION):
|
||||
self.id = obj_id
|
||||
self.uuid = uuid.uuid4()
|
||||
self.software_version = software_version
|
||||
|
||||
|
||||
class FakeUpgrade(object):
|
||||
def __init__(self,
|
||||
obj_id=1,
|
||||
state='completed',
|
||||
from_release=PREVIOUS_VERSION,
|
||||
to_release=UPGRADED_VERSION):
|
||||
self.id = obj_id
|
||||
self.uuid = uuid.uuid4()
|
||||
self.state = state
|
||||
self.from_release = from_release
|
||||
self.to_release = to_release
|
||||
self.links = []
|
||||
|
||||
|
||||
class FakeSysinvClient(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
@ -31,7 +73,7 @@ class FakeController(object):
|
|||
administrative=sysinv_constants.ADMIN_UNLOCKED,
|
||||
availability=sysinv_constants.AVAILABILITY_AVAILABLE,
|
||||
ihost_action=None,
|
||||
target_load=CURRENT_LOAD,
|
||||
target_load=UPGRADED_VERSION,
|
||||
task=None):
|
||||
self.id = host_id
|
||||
self.hostname = hostname
|
||||
|
|
|
@ -0,0 +1,90 @@
|
|||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
import mock
|
||||
|
||||
from dcmanager.common import consts
|
||||
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||
import TestSwUpgradeState
|
||||
|
||||
VALID_UPGRADE = FakeUpgrade(state='activation-complete')
|
||||
INVALID_UPGRADE = FakeUpgrade(state='aborting')
|
||||
|
||||
|
||||
class TestSwUpgradeCompletingStage(TestSwUpgradeState):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSwUpgradeCompletingStage, self).setUp()
|
||||
|
||||
# next state after completing an upgrade is 'complete'
|
||||
self.on_success_state = consts.STRATEGY_STATE_COMPLETE
|
||||
|
||||
# Add the strategy_step state being processed by this unit test
|
||||
self.strategy_step = \
|
||||
self.setup_strategy_step(consts.STRATEGY_STATE_COMPLETING_UPGRADE)
|
||||
|
||||
# Add mock API endpoints for sysinv client calls invcked by this state
|
||||
self.sysinv_client.upgrade_complete = mock.MagicMock()
|
||||
self.sysinv_client.get_upgrades = mock.MagicMock()
|
||||
|
||||
def test_upgrade_subcloud_completing_upgrade_failure(self):
|
||||
"""Test the completing upgrade API call fails."""
|
||||
|
||||
# upgrade_complete will only be called if an appropriate upgrade exists
|
||||
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
|
||||
|
||||
# API call raises an exception when it is rejected
|
||||
self.sysinv_client.upgrade_complete.side_effect = \
|
||||
Exception("upgrade complete failed for some reason")
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the expected API call was invoked
|
||||
self.sysinv_client.upgrade_complete.assert_called()
|
||||
|
||||
# Verify the state moves to 'failed'
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
def test_upgrade_subcloud_completing_upgrade_success(self):
|
||||
"""Test the completing upgrade step succeeds."""
|
||||
|
||||
# upgrade_complete will only be called if an appropriate upgrade exists
|
||||
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
|
||||
|
||||
# API call will not raise an exception. It will delete the upgrade
|
||||
self.sysinv_client.upgrade_complete.return_value = None
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the API cvall was invoked
|
||||
self.sysinv_client.upgrade_complete.assert_called()
|
||||
|
||||
# On success, the state should be updated to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
||||
|
||||
def test_upgrade_subcloud_completing_upgrade_skip_already_completed(self):
|
||||
"""Test the completing upgrade step skipped if already completed."""
|
||||
|
||||
# upgrade_complete will only be called if an appropriate upgrade exists
|
||||
# If the upgrade has been deleted, there is nothing to complete
|
||||
self.sysinv_client.get_upgrades.return_value = []
|
||||
|
||||
# API call will not be invoked, so no need to mock it
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# upgrade is already in one of the completing states so skip completing
|
||||
self.sysinv_client.upgrade_complete.assert_not_called()
|
||||
|
||||
# On success, the state is set to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
|
@ -0,0 +1,155 @@
|
|||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
import mock
|
||||
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.common.exceptions import VaultLoadMissingError
|
||||
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeLoad
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeSystem
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||
import PREVIOUS_VERSION
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||
import TestSwUpgradeState
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||
import UPGRADED_VERSION
|
||||
|
||||
PREVIOUS_LOAD = FakeLoad(1, software_version=PREVIOUS_VERSION)
|
||||
UPGRADED_LOAD = FakeLoad(2,
|
||||
compatible_version=PREVIOUS_VERSION,
|
||||
software_version=UPGRADED_VERSION)
|
||||
|
||||
DEST_LOAD_EXISTS = [PREVIOUS_LOAD, UPGRADED_LOAD, ]
|
||||
DEST_LOAD_MISSING = [PREVIOUS_LOAD, ]
|
||||
|
||||
FAKE_ISO = '/opt/dc-vault/loads/' + UPGRADED_VERSION + '/bootimage.iso'
|
||||
FAKE_SIG = '/opt/dc-vault/loads/' + UPGRADED_VERSION + '/bootimage.sig'
|
||||
|
||||
FAILED_IMPORT_RESPONSE = 'kaboom'
|
||||
SUCCESS_IMPORT_RESPONSE = {
|
||||
'new_load': {
|
||||
'id': 2,
|
||||
'uuid': 'aaa4b4c6-8536-41f6-87ea-211d208a723b',
|
||||
'compatible_version': PREVIOUS_VERSION,
|
||||
'required_patches': '',
|
||||
'software_version': UPGRADED_VERSION,
|
||||
'state': 'importing',
|
||||
'created_at': '2020-06-01 12:12:12+00:00',
|
||||
'updated_at': None
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class TestSwUpgradeImportingLoadStage(TestSwUpgradeState):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSwUpgradeImportingLoadStage, self).setUp()
|
||||
|
||||
# next state after 'importing load' is 'starting upgrade'
|
||||
self.on_success_state = consts.STRATEGY_STATE_STARTING_UPGRADE
|
||||
|
||||
# Add the strategy_step state being processed by this unit test
|
||||
self.strategy_step = \
|
||||
self.setup_strategy_step(consts.STRATEGY_STATE_IMPORTING_LOAD)
|
||||
|
||||
# Add mock API endpoints for sysinv client calls invcked by this state
|
||||
self.sysinv_client.get_system = mock.MagicMock()
|
||||
self.sysinv_client.get_system.return_value = FakeSystem()
|
||||
self.sysinv_client.get_loads = mock.MagicMock()
|
||||
self.sysinv_client.import_load = mock.MagicMock()
|
||||
|
||||
@mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files')
|
||||
def test_upgrade_subcloud_importing_load_failure(self, mock_vault_files):
|
||||
"""Test importing load step where the import_load API call fails."""
|
||||
|
||||
# simulate determine_matching_load finding the iso and sig in the vault
|
||||
mock_vault_files.return_value = (FAKE_ISO, FAKE_SIG)
|
||||
|
||||
# Simulate the target load has not been imported yet on the subcloud
|
||||
self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING
|
||||
|
||||
# Simulate an API failure on the subcloud.
|
||||
self.sysinv_client.import_load.return_value = FAILED_IMPORT_RESPONSE
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the import load API call was invoked
|
||||
self.sysinv_client.import_load.assert_called()
|
||||
|
||||
# Verify a failure leads to a state failure
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
@mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files')
|
||||
def test_upgrade_subcloud_importing_load_success(self, mock_vault_files):
|
||||
"""Test the importing load step succeeds.
|
||||
|
||||
The load will be imported on the subcloud when the subcloud does not
|
||||
have the load already imported, and the API call succeeds to import it.
|
||||
"""
|
||||
# simulate determine_matching_load finding the iso and sig in the vault
|
||||
mock_vault_files.return_value = (FAKE_ISO, FAKE_SIG)
|
||||
|
||||
# Simulate the target load has not been imported yet on the subcloud
|
||||
self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING
|
||||
|
||||
# Simulate an API success on the subcloud.
|
||||
self.sysinv_client.import_load.return_value = SUCCESS_IMPORT_RESPONSE
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the import load API call was invoked
|
||||
self.sysinv_client.import_load.assert_called()
|
||||
|
||||
# On success, should have moved to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
||||
|
||||
@mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files')
|
||||
def test_upgrade_subcloud_importing_load_fails_missing_vault_files(
|
||||
self,
|
||||
mock_determine_matching_load):
|
||||
"""Test importing load fails when files are not in the vault."""
|
||||
|
||||
mock_determine_matching_load.side_effect = \
|
||||
VaultLoadMissingError(file_type='.iso', vault_dir='/mock/vault/')
|
||||
|
||||
# Simulate the target load has not been imported yet on the subcloud
|
||||
self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING
|
||||
|
||||
# Simulate an API success on the subcloud. It should not get here.
|
||||
self.sysinv_client.import_load.return_value = SUCCESS_IMPORT_RESPONSE
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the import load API call was never invoked
|
||||
self.sysinv_client.import_load.assert_not_called()
|
||||
|
||||
# Verify a failure leads to a state failure
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
def test_upgrade_subcloud_importing_load_skip_existing(self):
|
||||
"""Test the importing load step skipped due to load already there"""
|
||||
|
||||
# Simulate the target load has been previously imported on the subcloud
|
||||
self.sysinv_client.get_loads.return_value = DEST_LOAD_EXISTS
|
||||
|
||||
# Simulate an API failure for import_load. It should not be called.
|
||||
self.sysinv_client.import_load.return_value = FAILED_IMPORT_RESPONSE
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# The import_load should not have been attempted
|
||||
self.sysinv_client.import_load.assert_not_called()
|
||||
|
||||
# On success, should have moved to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
|
@ -0,0 +1,158 @@
|
|||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
import mock
|
||||
|
||||
from dcmanager.common import consts
|
||||
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||
import TestSwUpgradeState
|
||||
|
||||
MISSING_LICENSE_RESPONSE = {
|
||||
u'content': u'',
|
||||
u'error': u'License file not found. A license may not have been installed.'
|
||||
}
|
||||
|
||||
LICENSE_VALID_RESPONSE = {
|
||||
u'content': u'A valid license',
|
||||
u'error': u''
|
||||
}
|
||||
|
||||
ALTERNATE_LICENSE_RESPONSE = {
|
||||
u'content': u'A different valid license',
|
||||
u'error': u''
|
||||
}
|
||||
|
||||
|
||||
class TestSwUpgradeInstallingLicenseStage(TestSwUpgradeState):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSwUpgradeInstallingLicenseStage, self).setUp()
|
||||
|
||||
# next state after installing a license is 'importing load'
|
||||
self.on_success_state = consts.STRATEGY_STATE_IMPORTING_LOAD
|
||||
|
||||
# Add the strategy_step state being processed by this unit test
|
||||
self.strategy_step = \
|
||||
self.setup_strategy_step(consts.STRATEGY_STATE_INSTALLING_LICENSE)
|
||||
|
||||
# Add mock API endpoints for sysinv client calls invcked by this state
|
||||
self.sysinv_client.get_license = mock.MagicMock()
|
||||
self.sysinv_client.install_license = mock.MagicMock()
|
||||
|
||||
def test_upgrade_subcloud_license_install_failure(self):
|
||||
"""Test the installing license step where the install fails.
|
||||
|
||||
The system controller has a license, but the API call to install on the
|
||||
subcloud fails.
|
||||
"""
|
||||
|
||||
# Order of get_license calls:
|
||||
# first license query is to system controller
|
||||
# second license query is to subcloud (should be missing)
|
||||
self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE,
|
||||
MISSING_LICENSE_RESPONSE]
|
||||
|
||||
# Simulate a license install failure on the subcloud
|
||||
self.sysinv_client.install_license.return_value = \
|
||||
MISSING_LICENSE_RESPONSE
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the license install was invoked
|
||||
self.sysinv_client.install_license.assert_called()
|
||||
|
||||
# Verify a install_license failure leads to a state failure
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
def test_upgrade_subcloud_license_install_success(self):
|
||||
"""Test the install license step succeeds.
|
||||
|
||||
The license will be installed on the subcloud when system controller
|
||||
has a license, the subcloud does not have a license, and the API call
|
||||
succeeds.
|
||||
"""
|
||||
|
||||
# Order of get_license calls:
|
||||
# first license query is to system controller
|
||||
# second license query is to subcloud (should be missing)
|
||||
self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE,
|
||||
MISSING_LICENSE_RESPONSE]
|
||||
|
||||
# A license install should return a success
|
||||
self.sysinv_client.install_license.return_value = \
|
||||
LICENSE_VALID_RESPONSE
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the license install was invoked
|
||||
self.sysinv_client.install_license.assert_called()
|
||||
|
||||
# On success, the next state after installing license is importing load
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
||||
|
||||
def test_upgrade_subcloud_license_skip_existing(self):
|
||||
"""Test the install license step skipped due to license up to date"""
|
||||
|
||||
# Order of get_license calls:
|
||||
# first license query is to system controller
|
||||
# second license query is to subcloud
|
||||
self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE,
|
||||
LICENSE_VALID_RESPONSE]
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# A license install should not have been attempted due to the license
|
||||
# already being up to date
|
||||
self.sysinv_client.install_license.assert_not_called()
|
||||
|
||||
# On success, the next state after installing license is importing load
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
||||
|
||||
def test_upgrade_subcloud_license_overrides_mismatched_license(self):
|
||||
"""Test the install license overrides a mismatched license"""
|
||||
|
||||
# Order of get_license calls:
|
||||
# first license query is to system controller
|
||||
# second license query is to subcloud (should be valid but different)
|
||||
self.sysinv_client.get_license.side_effect = \
|
||||
[LICENSE_VALID_RESPONSE,
|
||||
ALTERNATE_LICENSE_RESPONSE]
|
||||
|
||||
# A license install should return a success
|
||||
self.sysinv_client.install_license.return_value = \
|
||||
LICENSE_VALID_RESPONSE
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the license install was invoked
|
||||
self.sysinv_client.install_license.assert_called()
|
||||
|
||||
# Verify it successfully moves to the next step
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
||||
|
||||
def test_upgrade_subcloud_license_skip_when_no_sys_controller_lic(self):
|
||||
"""Test license install skipped when no license on system controller."""
|
||||
|
||||
# Only makes one query: to system controller
|
||||
self.sysinv_client.get_license.return_value = MISSING_LICENSE_RESPONSE
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# Should skip install_license API call
|
||||
self.sysinv_client.install_license.assert_not_called()
|
||||
|
||||
# Verify it successfully moves to the next step
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
|
@ -56,8 +56,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
|
|||
# mock the API call as failed on the subcloud
|
||||
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING
|
||||
|
||||
# invoke the strategy state operation
|
||||
self.worker.lock_subcloud_controller(self.strategy_step)
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the lock command was actually attempted
|
||||
self.sysinv_client.lock_host.assert_called()
|
||||
|
@ -72,8 +72,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
|
|||
# mock the controller host query as being already locked
|
||||
self.sysinv_client.get_host.return_value = CONTROLLER_0_LOCKED
|
||||
|
||||
# invoke the strategy state operation
|
||||
self.worker.lock_subcloud_controller(self.strategy_step)
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the lock command was never attempted
|
||||
self.sysinv_client.lock_host.assert_not_called()
|
||||
|
@ -95,8 +95,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
|
|||
# mock the API call as successful on the subcloud
|
||||
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING
|
||||
|
||||
# invoke the strategy state operation
|
||||
self.worker.lock_subcloud_controller(self.strategy_step)
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the lock command was actually attempted
|
||||
self.sysinv_client.lock_host.assert_called()
|
||||
|
@ -118,8 +118,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
|
|||
# mock the API call as failed on the subcloud
|
||||
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING_FAILED
|
||||
|
||||
# invoke the strategy state operation
|
||||
self.worker.lock_subcloud_controller(self.strategy_step)
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the lock command was actually attempted
|
||||
self.sysinv_client.lock_host.assert_called()
|
||||
|
@ -135,8 +135,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
|
|||
self.sysinv_client.get_host.side_effect = \
|
||||
Exception("Unable to find host controller-0")
|
||||
|
||||
# invoke the strategy state operation
|
||||
self.worker.lock_subcloud_controller(self.strategy_step)
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the lock command was never attempted
|
||||
self.sysinv_client.lock_host.assert_not_called()
|
||||
|
|
|
@ -0,0 +1,124 @@
|
|||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
import mock
|
||||
|
||||
from dcmanager.common import consts
|
||||
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade
|
||||
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||
import TestSwUpgradeState
|
||||
|
||||
UPGRADE_ABORTING = [FakeUpgrade(state='aborting'), ]
|
||||
UPGRADE_STARTED = [FakeUpgrade(state='started'), ]
|
||||
SUCCESS_UPGRADE_START = 'I do not know what this looks like yet'
|
||||
|
||||
|
||||
class TestSwUpgradeStartingUpgradeStage(TestSwUpgradeState):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSwUpgradeStartingUpgradeStage, self).setUp()
|
||||
|
||||
# next state after 'starting upgrade' is 'migrating data'
|
||||
self.on_success_state = consts.STRATEGY_STATE_LOCKING_CONTROLLER
|
||||
|
||||
# Add the strategy_step state being processed by this unit test
|
||||
self.strategy_step = \
|
||||
self.setup_strategy_step(consts.STRATEGY_STATE_STARTING_UPGRADE)
|
||||
|
||||
# Add mock API endpoints for sysinv client calls invcked by this state
|
||||
self.sysinv_client.upgrade_start = mock.MagicMock()
|
||||
self.sysinv_client.get_upgrades = mock.MagicMock()
|
||||
|
||||
def test_upgrade_subcloud_upgrade_start_failure(self):
|
||||
"""Test the upgrade_start where the API call fails.
|
||||
|
||||
The upgrade_start call fails due to a validation check such as from
|
||||
the health-query check.
|
||||
"""
|
||||
|
||||
# No upgrades should yet exist in the DB / API
|
||||
self.sysinv_client.get_upgrades.return_value = []
|
||||
|
||||
# Simulate an upgrade_start failure on the subcloud.
|
||||
# The API throws an exception rather than returning an error response
|
||||
self.sysinv_client.upgrade_start.side_effect = \
|
||||
Exception("HTTPBadRequest: upgrade-start rejected: "
|
||||
"System is not in a valid state for upgrades. "
|
||||
"Run system health-query-upgrade for more details.")
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the API call that failed was invoked
|
||||
self.sysinv_client.upgrade_start.assert_called()
|
||||
|
||||
# Verify the API failure leads to a state failure
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
def test_upgrade_subcloud_upgrade_start_success(self):
|
||||
"""Test upgrade_start where the API call succeeds.
|
||||
|
||||
This will result in an upgrade being created with the appropriate
|
||||
state.
|
||||
"""
|
||||
|
||||
# No upgrades should yet exist in the DB / API
|
||||
self.sysinv_client.get_upgrades.return_value = []
|
||||
|
||||
# Simulate an upgrade_start succeeds on the subcloud
|
||||
self.sysinv_client.upgrade_start.return_value = SUCCESS_UPGRADE_START
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the API call that succeeded was actually invoked
|
||||
self.sysinv_client.upgrade_start.assert_called()
|
||||
|
||||
# On success, the state should transition to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
||||
|
||||
def test_upgrade_subcloud_upgrade_start_skip_already_started(self):
|
||||
"""Test upgrade_start where the upgrade is already started."""
|
||||
|
||||
# An already started upgrade exists in the DB"""
|
||||
self.sysinv_client.get_upgrades.return_value = [UPGRADE_STARTED, ]
|
||||
|
||||
# upgrade_start should not be invoked, so can be mocked as 'failed'
|
||||
# by raising an exception
|
||||
self.sysinv_client.upgrade_start.side_effect = \
|
||||
Exception("HTTPBadRequest: this is a fake exception")
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# upgrade_start API call should not have been attempted due to the
|
||||
# existing upgrade already in started state.
|
||||
self.sysinv_client.upgrade_start.assert_not_called()
|
||||
|
||||
# On success, the state should transition to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
||||
|
||||
def test_upgrade_subcloud_upgrade_start_fails_bad_existing_upgrade(self):
|
||||
"""Test the upgrade_start fails due to a bad existing upgrade."""
|
||||
|
||||
# An already started upgrade exists in the DB but is in bad shape."""
|
||||
self.sysinv_client.get_upgrades.return_value = [UPGRADE_ABORTING, ]
|
||||
|
||||
# upgrade_start will NOT be invoked. No need to mock it.
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# upgrade_start API call should not have been attempted due to the
|
||||
# invalid existing upgrade that needs to be cleaned up.
|
||||
self.sysinv_client.upgrade_start.assert_not_called()
|
||||
|
||||
# Verify it failed and moves to the next step
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.on_success_state)
|
|
@ -31,7 +31,7 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||
super(TestSwUpgradeUnlockControllerStage, self).setUp()
|
||||
|
||||
# next state after a successful unlock is 'activating'
|
||||
self.on_success_state = consts.STRATEGY_STATE_ACTIVATING
|
||||
self.on_success_state = consts.STRATEGY_STATE_ACTIVATING_UPGRADE
|
||||
|
||||
# Add the strategy_step state being processed by this unit test
|
||||
self.strategy_step = self.setup_strategy_step(
|
||||
|
@ -56,8 +56,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||
# mock the API call as failed on the subcloud
|
||||
self.sysinv_client.unlock_host.return_value = CONTROLLER_0_UNLOCKING
|
||||
|
||||
# invoke the strategy state operation
|
||||
self.worker.unlock_subcloud_controller(self.strategy_step)
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the unlock command was actually attempted
|
||||
self.sysinv_client.unlock_host.assert_called()
|
||||
|
@ -72,8 +72,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||
# mock the controller host query as being already unlocked
|
||||
self.sysinv_client.get_host.return_value = CONTROLLER_0_UNLOCKED
|
||||
|
||||
# invoke the strategy state operation
|
||||
self.worker.unlock_subcloud_controller(self.strategy_step)
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the unlock command was never attempted
|
||||
self.sysinv_client.unlock_host.assert_not_called()
|
||||
|
@ -95,8 +95,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||
# mock the API call as successful on the subcloud
|
||||
self.sysinv_client.unlock_host.return_value = CONTROLLER_0_UNLOCKING
|
||||
|
||||
# invoke the strategy state operation
|
||||
self.worker.unlock_subcloud_controller(self.strategy_step)
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the lock command was actually attempted
|
||||
self.sysinv_client.unlock_host.assert_called()
|
||||
|
@ -119,8 +119,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||
self.sysinv_client.unlock_host.return_value = \
|
||||
CONTROLLER_0_UNLOCKING_FAILED
|
||||
|
||||
# invoke the strategy state operation
|
||||
self.worker.unlock_subcloud_controller(self.strategy_step)
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the unlock command was actually attempted
|
||||
self.sysinv_client.unlock_host.assert_called()
|
||||
|
@ -136,8 +136,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||
self.sysinv_client.get_host.side_effect = \
|
||||
Exception("Unable to find host controller-0")
|
||||
|
||||
# invoke the strategy state operation
|
||||
self.worker.unlock_subcloud_controller(self.strategy_step)
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the unlock command was never attempted
|
||||
self.sysinv_client.unlock_host.assert_not_called()
|
||||
|
|
|
@ -33,7 +33,6 @@ from dcmanager.tests.unit.manager.test_sw_update_manager \
|
|||
import Subcloud
|
||||
from dcmanager.tests import utils
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
FAKE_ID = '1'
|
||||
FAKE_SW_UPDATE_DATA = {
|
||||
|
@ -53,29 +52,6 @@ FAKE_STRATEGY_STEP_DATA = {
|
|||
"subcloud": None
|
||||
}
|
||||
|
||||
MISSING_LICENSE_RESPONSE = {
|
||||
u'content': u'',
|
||||
u'error': u'License file not found. A license may not have been installed.'
|
||||
}
|
||||
|
||||
LICENSE_VALID_RESPONSE = {
|
||||
u'content': u'A valid license',
|
||||
u'error': u''
|
||||
}
|
||||
|
||||
ALTERNATE_LICENSE_RESPONSE = {
|
||||
u'content': u'A different valid license',
|
||||
u'error': u''
|
||||
}
|
||||
|
||||
|
||||
class FakeSysinvClient(object):
|
||||
|
||||
def __init__(self):
|
||||
super(FakeSysinvClient, self).__init__()
|
||||
self.get_license = mock.MagicMock()
|
||||
self.install_license = mock.MagicMock()
|
||||
|
||||
|
||||
class TestSwUpgrade(base.DCManagerTestCase):
|
||||
def setUp(self):
|
||||
|
@ -99,14 +75,6 @@ class TestSwUpgrade(base.DCManagerTestCase):
|
|||
self.fake_patch_orch_thread
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
# Mock the sysinv client
|
||||
self.fake_sysinv_client = FakeSysinvClient()
|
||||
p = mock.patch.object(sw_upgrade_orch_thread.SwUpgradeOrchThread,
|
||||
'get_sysinv_client')
|
||||
self.mock_sysinv_client = p.start()
|
||||
self.mock_sysinv_client.return_value = self.fake_sysinv_client
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
# Mock db_api
|
||||
p = mock.patch.object(sw_upgrade_orch_thread, 'db_api')
|
||||
self.mock_db_api = p.start()
|
||||
|
@ -128,7 +96,6 @@ class TestSwUpgrade(base.DCManagerTestCase):
|
|||
mock_dcmanager_audit_api = mock.Mock()
|
||||
worker = sw_update_manager.SwUpgradeOrchThread(mock_strategy_lock,
|
||||
mock_dcmanager_audit_api)
|
||||
worker.get_ks_client = mock.Mock()
|
||||
return worker
|
||||
|
||||
def assert_step_updated(self, subcloud_id, update_state):
|
||||
|
@ -140,168 +107,3 @@ class TestSwUpgrade(base.DCManagerTestCase):
|
|||
started_at=mock.ANY,
|
||||
finished_at=mock.ANY,
|
||||
)
|
||||
|
||||
|
||||
class TestSwUpgradeLicenseStage(TestSwUpgrade):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSwUpgradeLicenseStage, self).setUp()
|
||||
self.strategy_step = \
|
||||
self.setup_strategy_step(consts.STRATEGY_STATE_INSTALLING_LICENSE)
|
||||
|
||||
def test_upgrade_subcloud_license_install_failure(self):
|
||||
# Test the install subcloud license step where the system controller
|
||||
# license is valid, and the subcloud license install fails
|
||||
|
||||
# Order of get_license calls:
|
||||
# first license query is to system controller
|
||||
# second license query is to subcloud (should be missing)
|
||||
self.fake_sysinv_client.get_license.side_effect = \
|
||||
[LICENSE_VALID_RESPONSE,
|
||||
MISSING_LICENSE_RESPONSE]
|
||||
|
||||
# Simulate a license install failure on the subcloud
|
||||
self.fake_sysinv_client.install_license.return_value = \
|
||||
MISSING_LICENSE_RESPONSE
|
||||
|
||||
self.worker.install_subcloud_license(self.strategy_step)
|
||||
|
||||
# verify the license install was invoked
|
||||
self.fake_sysinv_client.install_license.assert_called()
|
||||
|
||||
# Verify a install_license failure leads to a state failure
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
def test_upgrade_subcloud_license_install_success(self):
|
||||
# Test the install subcloud license step where the system controller
|
||||
# license is valid, and the subcloud installation succeeds
|
||||
|
||||
# Order of get_license calls:
|
||||
# first license query is to system controller
|
||||
# second license query is to subcloud (should be missing)
|
||||
self.fake_sysinv_client.get_license.side_effect = \
|
||||
[LICENSE_VALID_RESPONSE,
|
||||
MISSING_LICENSE_RESPONSE]
|
||||
|
||||
# A license install should return a success
|
||||
self.fake_sysinv_client.install_license.return_value = \
|
||||
LICENSE_VALID_RESPONSE
|
||||
|
||||
self.worker.install_subcloud_license(self.strategy_step)
|
||||
|
||||
# verify the license install was invoked
|
||||
self.fake_sysinv_client.install_license.assert_called()
|
||||
|
||||
# On success, the next state after installing license is importing load
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_IMPORTING_LOAD)
|
||||
|
||||
def test_upgrade_subcloud_license_skip_existing(self):
|
||||
# Test the install subcloud license step where the system controller
|
||||
# license is valid, and the subcloud already has the same license
|
||||
|
||||
# Order of get_license calls:
|
||||
# first license query is to system controller
|
||||
# second license query is to subcloud
|
||||
self.fake_sysinv_client.get_license.side_effect = \
|
||||
[LICENSE_VALID_RESPONSE,
|
||||
LICENSE_VALID_RESPONSE]
|
||||
self.worker.install_subcloud_license(self.strategy_step)
|
||||
|
||||
# A license install should not have been attempted due to the license
|
||||
# already being up to date
|
||||
self.fake_sysinv_client.install_license.assert_not_called()
|
||||
# On success, the next state after installing license is importing load
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_IMPORTING_LOAD)
|
||||
|
||||
def test_upgrade_subcloud_license_overrides_mismatched_license(self):
|
||||
# Test the install subcloud license step where the system controller
|
||||
# license is valid, and the subcloud has a differnt license which
|
||||
# should be overridden
|
||||
|
||||
# Order of get_license calls:
|
||||
# first license query is to system controller
|
||||
# second license query is to subcloud (should be valid but different)
|
||||
self.fake_sysinv_client.get_license.side_effect = \
|
||||
[LICENSE_VALID_RESPONSE,
|
||||
ALTERNATE_LICENSE_RESPONSE]
|
||||
|
||||
# A license install should return a success
|
||||
self.fake_sysinv_client.install_license.return_value = \
|
||||
LICENSE_VALID_RESPONSE
|
||||
|
||||
self.worker.install_subcloud_license(self.strategy_step)
|
||||
|
||||
# verify the license install was invoked
|
||||
self.fake_sysinv_client.install_license.assert_called()
|
||||
|
||||
# On success, the next state after installing license is importing load
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_IMPORTING_LOAD)
|
||||
|
||||
def test_upgrade_subcloud_license_skip_when_no_sys_controller_lic(self):
|
||||
# Test the install subcloud license step is skipped and proceeds
|
||||
# to the next state when there is no license on system controller
|
||||
|
||||
# Only makes one query to system controller
|
||||
self.fake_sysinv_client.get_license.side_effect = \
|
||||
[MISSING_LICENSE_RESPONSE, ]
|
||||
# Test the install subcloud license stage
|
||||
self.worker.install_subcloud_license(self.strategy_step)
|
||||
|
||||
# A license install should proceed to the next state without
|
||||
# calling a license install
|
||||
self.fake_sysinv_client.install_license.assert_not_called()
|
||||
# Skip license install and move to next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_IMPORTING_LOAD)
|
||||
|
||||
def test_upgrade_subcloud_license_handle_failure(self):
|
||||
# Test the install subcloud license step where the system controller
|
||||
# license is valid, and the subcloud license install fails
|
||||
|
||||
# Order of get_license calls:
|
||||
# first license query is to system controller
|
||||
# second license query is to subcloud (should be missing)
|
||||
self.fake_sysinv_client.get_license.side_effect = \
|
||||
[LICENSE_VALID_RESPONSE,
|
||||
MISSING_LICENSE_RESPONSE]
|
||||
|
||||
# Simulate a license install failure on the subcloud
|
||||
self.fake_sysinv_client.install_license.return_value = \
|
||||
MISSING_LICENSE_RESPONSE
|
||||
|
||||
self.worker.install_subcloud_license(self.strategy_step)
|
||||
|
||||
# verify the license install was invoked
|
||||
self.fake_sysinv_client.install_license.assert_called()
|
||||
|
||||
# Verify a install_license failure leads to a state failure
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
def test_upgrade_subcloud_license_installs(self):
|
||||
# Test the install subcloud license step where the system controller
|
||||
# license is valid, and the subcloud installation succeeds
|
||||
|
||||
# Order of get_license calls:
|
||||
# first license query is to system controller
|
||||
# second license query is to subcloud (should be missing)
|
||||
self.fake_sysinv_client.get_license.side_effect = \
|
||||
[LICENSE_VALID_RESPONSE,
|
||||
MISSING_LICENSE_RESPONSE]
|
||||
|
||||
# A license install should return a success
|
||||
self.fake_sysinv_client.install_license.return_value = \
|
||||
LICENSE_VALID_RESPONSE
|
||||
|
||||
self.worker.install_subcloud_license(self.strategy_step)
|
||||
|
||||
# verify the license install was invoked
|
||||
self.fake_sysinv_client.install_license.assert_called()
|
||||
|
||||
# On success, the next state after installing license is importing load
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_IMPORTING_LOAD)
|
||||
|
|
Loading…
Reference in New Issue